Model save

README.md

@@ -1,13 +1,11 @@
 ---
+license: apache-2.0
 library_name: peft
 tags:
-- alignment-handbook
 - trl
 - dpo
 - generated_from_trainer
 base_model: norallm/normistral-7b-warm
-datasets:
-- hugodk-sch/aftonposten_title_prefs
 model-index:
 - name: ap-normistral-7b-align-scan
   results: []
@@ -18,17 +16,17 @@ should probably proofread and complete it, then remove this comment. -->
 
 # ap-normistral-7b-align-scan
 
-This model is a fine-tuned version of [data/ap-normistral-7b-sft-qlora](https://huggingface.co/data/ap-normistral-7b-sft-qlora) on the hugodk-sch/aftonposten_title_prefs dataset.
+This model is a fine-tuned version of [norallm/normistral-7b-warm](https://huggingface.co/norallm/normistral-7b-warm) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 1.6647
-- Rewards/chosen: 0.0868
-- Rewards/rejected: 0.0880
-- Rewards/accuracies: 0.5108
-- Rewards/margins: -0.0012
-- Logps/rejected: -35.8687
-- Logps/chosen: -32.3467
-- Logits/rejected: 99.0734
-- Logits/chosen: 99.0806
+- Loss: 1.0329
+- Rewards/chosen: -0.0989
+- Rewards/rejected: -0.1202
+- Rewards/accuracies: 0.5457
+- Rewards/margins: 0.0213
+- Logps/rejected: -36.1669
+- Logps/chosen: -32.6081
+- Logits/rejected: 98.7340
+- Logits/chosen: 98.7641
 
 ## Model description
 
@@ -63,9 +61,9 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
 |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
-| 2.0775        | 0.26  | 100  | 1.7337          | -0.0836        | -0.0209          | 0.4747             | -0.0626         | -35.9898       | -32.5360     | 98.9624         | 98.9691       |
-| 3.1221        | 0.52  | 200  | 1.7165          | -0.0706        | -0.1245          | 0.5278             | 0.0539          | -36.1048       | -32.5217     | 99.1580         | 99.1691       |
-| 2.4404        | 0.78  | 300  | 1.5174          | 0.1203         | 0.1230           | 0.5108             | -0.0027         | -35.8299       | -32.3095     | 99.0568         | 99.0617       |
+| 0.8073        | 0.26  | 100  | 1.0338          | 0.0248         | 0.0277           | 0.4967             | -0.0029         | -35.9204       | -32.4018     | 98.7406         | 98.7512       |
+| 0.7227        | 0.52  | 200  | 1.0227          | -0.1121        | -0.1359          | 0.4950             | 0.0238          | -36.1930       | -32.6300     | 98.7528         | 98.7726       |
+| 0.6355        | 0.78  | 300  | 1.0329          | -0.0989        | -0.1202          | 0.5457             | 0.0213          | -36.1669       | -32.6081     | 98.7340         | 98.7641       |
 
 
 ### Framework versions

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:96b90d15e40eb860590d62e9dd0f47fa5c12fbb3c0f9e48c03f28940363c7e3a
+oid sha256:324e65a254fc1ae88a61def4f1589384315581f3cb7a6c68e9d63b4228bf265e
 size 671150064

all_results.json

@@ -1,21 +1,8 @@
 {
     "epoch": 1.0,
-    "eval_logits/chosen": 99.08063507080078,
-    "eval_logits/rejected": 99.07344818115234,
-    "eval_logps/chosen": -32.34668731689453,
-    "eval_logps/rejected": -35.868736267089844,
-    "eval_loss": 1.664738655090332,
-    "eval_rewards/accuracies": 0.510797381401062,
-    "eval_rewards/chosen": 0.08684198558330536,
-    "eval_rewards/margins": -0.0011658848961815238,
-    "eval_rewards/rejected": 0.08800788223743439,
-    "eval_runtime": 103.8246,
-    "eval_samples": 343,
-    "eval_samples_per_second": 3.304,
-    "eval_steps_per_second": 0.414,
-    "train_loss": 2.2499145250041765,
-    "train_runtime": 2558.7273,
+    "train_loss": 0.746250953302755,
+    "train_runtime": 2556.1232,
     "train_samples": 3079,
-    "train_samples_per_second": 1.203,
-    "train_steps_per_second": 0.15
+    "train_samples_per_second": 1.205,
+    "train_steps_per_second": 0.151
 }

train_results.json

@@ -1,8 +1,8 @@
 {
     "epoch": 1.0,
-    "train_loss": 2.2499145250041765,
-    "train_runtime": 2558.7273,
+    "train_loss": 0.746250953302755,
+    "train_runtime": 2556.1232,
     "train_samples": 3079,
-    "train_samples_per_second": 1.203,
-    "train_steps_per_second": 0.15
+    "train_samples_per_second": 1.205,
+    "train_steps_per_second": 0.151
 }

trainer_state.json

@@ -10,13 +10,13 @@
   "log_history": [
     {
       "epoch": 0.0,
-      "grad_norm": 145.0,
+      "grad_norm": 78.5,
       "learning_rate": 1.282051282051282e-07,
       "logits/chosen": 88.18099975585938,
       "logits/rejected": 88.25153350830078,
       "logps/chosen": -29.073104858398438,
       "logps/rejected": -26.25731658935547,
-      "loss": 0.3086,
+      "loss": 1.0,
       "rewards/accuracies": 0.0,
       "rewards/chosen": 0.0,
       "rewards/margins": 0.0,
@@ -25,630 +25,630 @@
     },
     {
       "epoch": 0.03,
-      "grad_norm": 288.0,
+      "grad_norm": 66.0,
       "learning_rate": 1.282051282051282e-06,
-      "logits/chosen": 81.08262634277344,
-      "logits/rejected": 80.7869873046875,
-      "logps/chosen": -34.28562545776367,
-      "logps/rejected": -33.03427505493164,
-      "loss": 0.9415,
-      "rewards/accuracies": 0.4861111044883728,
-      "rewards/chosen": -0.039508190006017685,
-      "rewards/margins": 0.02877185121178627,
-      "rewards/rejected": -0.06828003376722336,
+      "logits/chosen": 81.06854248046875,
+      "logits/rejected": 80.77387237548828,
+      "logps/chosen": -34.293479919433594,
+      "logps/rejected": -32.99631118774414,
+      "loss": 1.0126,
+      "rewards/accuracies": 0.4305555522441864,
+      "rewards/chosen": -0.03105219267308712,
+      "rewards/margins": -0.008308152668178082,
+      "rewards/rejected": -0.022744040936231613,
       "step": 10
     },
     {
       "epoch": 0.05,
-      "grad_norm": 149.0,
+      "grad_norm": 67.0,
       "learning_rate": 2.564102564102564e-06,
-      "logits/chosen": 80.65168762207031,
-      "logits/rejected": 80.53875732421875,
-      "logps/chosen": -33.57862091064453,
-      "logps/rejected": -30.82345199584961,
-      "loss": 1.1289,
+      "logits/chosen": 80.62992095947266,
+      "logits/rejected": 80.52108001708984,
+      "logps/chosen": -33.51418685913086,
+      "logps/rejected": -30.840993881225586,
+      "loss": 0.8773,
       "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.1001572236418724,
-      "rewards/margins": 0.14152315258979797,
-      "rewards/rejected": -0.041365914046764374,
+      "rewards/chosen": 0.10543360561132431,
+      "rewards/margins": 0.14353612065315247,
+      "rewards/rejected": -0.038102518767118454,
       "step": 20
     },
     {
       "epoch": 0.08,
-      "grad_norm": 234.0,
+      "grad_norm": 68.0,
       "learning_rate": 3.846153846153847e-06,
-      "logits/chosen": 82.32148742675781,
-      "logits/rejected": 82.35160827636719,
-      "logps/chosen": -33.95701599121094,
-      "logps/rejected": -31.341405868530273,
-      "loss": 1.3887,
-      "rewards/accuracies": 0.4375,
-      "rewards/chosen": 0.05022481083869934,
-      "rewards/margins": -0.023957695811986923,
-      "rewards/rejected": 0.07418251037597656,
+      "logits/chosen": 82.46273803710938,
+      "logits/rejected": 82.4950180053711,
+      "logps/chosen": -33.98906707763672,
+      "logps/rejected": -31.265905380249023,
+      "loss": 1.1079,
+      "rewards/accuracies": 0.44999998807907104,
+      "rewards/chosen": 0.014249947853386402,
+      "rewards/margins": -0.08050690591335297,
+      "rewards/rejected": 0.09475686401128769,
       "step": 30
     },
     {
       "epoch": 0.1,
-      "grad_norm": 636.0,
+      "grad_norm": 55.75,
       "learning_rate": 4.999896948438434e-06,
-      "logits/chosen": 80.53231811523438,
-      "logits/rejected": 80.530029296875,
-      "logps/chosen": -33.09763717651367,
-      "logps/rejected": -33.38296890258789,
-      "loss": 2.1646,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.06877875328063965,
-      "rewards/margins": 0.0817752406001091,
-      "rewards/rejected": -0.0129964929074049,
+      "logits/chosen": 80.95271301269531,
+      "logits/rejected": 80.94709777832031,
+      "logps/chosen": -32.69623565673828,
+      "logps/rejected": -33.20489501953125,
+      "loss": 0.8832,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.2866959571838379,
+      "rewards/margins": 0.1885182112455368,
+      "rewards/rejected": 0.09817773848772049,
       "step": 40
     },
     {
       "epoch": 0.13,
-      "grad_norm": 386.0,
+      "grad_norm": 37.5,
       "learning_rate": 4.987541037542187e-06,
-      "logits/chosen": 78.10733032226562,
-      "logits/rejected": 78.12706756591797,
-      "logps/chosen": -31.247085571289062,
-      "logps/rejected": -31.239765167236328,
-      "loss": 1.232,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": -0.03765222057700157,
-      "rewards/margins": 0.11217772960662842,
-      "rewards/rejected": -0.14982998371124268,
+      "logits/chosen": 78.52803039550781,
+      "logits/rejected": 78.53800964355469,
+      "logps/chosen": -30.692174911499023,
+      "logps/rejected": -30.75704002380371,
+      "loss": 1.0095,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": 0.30784496665000916,
+      "rewards/margins": 0.1180974617600441,
+      "rewards/rejected": 0.18974749743938446,
       "step": 50
     },
     {
       "epoch": 0.16,
-      "grad_norm": 200.0,
+      "grad_norm": 53.0,
       "learning_rate": 4.954691471941119e-06,
-      "logits/chosen": 82.77879333496094,
-      "logits/rejected": 82.82901763916016,
-      "logps/chosen": -31.19879150390625,
-      "logps/rejected": -29.63169288635254,
-      "loss": 1.1982,
-      "rewards/accuracies": 0.5249999761581421,
-      "rewards/chosen": -0.021271925419569016,
-      "rewards/margins": 0.015784021466970444,
-      "rewards/rejected": -0.03705594688653946,
+      "logits/chosen": 83.14569854736328,
+      "logits/rejected": 83.19773864746094,
+      "logps/chosen": -30.899333953857422,
+      "logps/rejected": -29.450210571289062,
+      "loss": 0.9563,
+      "rewards/accuracies": 0.512499988079071,
+      "rewards/chosen": 0.16549696028232574,
+      "rewards/margins": 0.08130919188261032,
+      "rewards/rejected": 0.08418775349855423,
       "step": 60
     },
     {
       "epoch": 0.18,
-      "grad_norm": 832.0,
+      "grad_norm": 69.5,
       "learning_rate": 4.901618883413549e-06,
-      "logits/chosen": 83.74064636230469,
-      "logits/rejected": 83.76899719238281,
-      "logps/chosen": -30.796112060546875,
-      "logps/rejected": -33.087310791015625,
-      "loss": 2.0455,
-      "rewards/accuracies": 0.44999998807907104,
-      "rewards/chosen": -0.15038001537322998,
-      "rewards/margins": -0.10573381185531616,
-      "rewards/rejected": -0.04464619606733322,
+      "logits/chosen": 83.82180786132812,
+      "logits/rejected": 83.84942626953125,
+      "logps/chosen": -30.46747398376465,
+      "logps/rejected": -33.01859664916992,
+      "loss": 0.9812,
+      "rewards/accuracies": 0.48750001192092896,
+      "rewards/chosen": 0.09693069756031036,
+      "rewards/margins": 0.08546572178602219,
+      "rewards/rejected": 0.011464978568255901,
       "step": 70
     },
     {
       "epoch": 0.21,
-      "grad_norm": 432.0,
+      "grad_norm": 56.25,
       "learning_rate": 4.828760511501322e-06,
-      "logits/chosen": 81.53680419921875,
-      "logits/rejected": 81.53197479248047,
-      "logps/chosen": -31.50222396850586,
-      "logps/rejected": -30.938213348388672,
-      "loss": 2.2722,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.0024462162982672453,
-      "rewards/margins": 0.16495725512504578,
-      "rewards/rejected": -0.16251103579998016,
+      "logits/chosen": 81.54592895507812,
+      "logits/rejected": 81.53177642822266,
+      "logps/chosen": -31.3655948638916,
+      "logps/rejected": -30.929000854492188,
+      "loss": 0.9098,
+      "rewards/accuracies": 0.5375000238418579,
+      "rewards/chosen": 0.0836077481508255,
+      "rewards/margins": 0.1864190548658371,
+      "rewards/rejected": -0.102811299264431,
       "step": 80
     },
     {
       "epoch": 0.23,
-      "grad_norm": 378.0,
+      "grad_norm": 71.5,
       "learning_rate": 4.7367166013034295e-06,
-      "logits/chosen": 78.6462173461914,
-      "logits/rejected": 78.61238098144531,
-      "logps/chosen": -32.54735565185547,
-      "logps/rejected": -30.98733139038086,
-      "loss": 1.7909,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.0817408338189125,
-      "rewards/margins": 0.11401765048503876,
-      "rewards/rejected": -0.03227682039141655,
+      "logits/chosen": 78.40837097167969,
+      "logits/rejected": 78.37776947021484,
+      "logps/chosen": -32.40131378173828,
+      "logps/rejected": -30.98993492126465,
+      "loss": 0.913,
+      "rewards/accuracies": 0.574999988079071,
+      "rewards/chosen": 0.14211884140968323,
+      "rewards/margins": 0.16519510746002197,
+      "rewards/rejected": -0.023076282814145088,
       "step": 90
     },
     {
       "epoch": 0.26,
-      "grad_norm": 210.0,
+      "grad_norm": 69.0,
       "learning_rate": 4.626245458345211e-06,
-      "logits/chosen": 84.08260345458984,
-      "logits/rejected": 84.1148910522461,
-      "logps/chosen": -34.308895111083984,
-      "logps/rejected": -31.754663467407227,
-      "loss": 2.0775,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.003516471479088068,
-      "rewards/margins": 0.00018945932970382273,
-      "rewards/rejected": -0.0037059492897242308,
+      "logits/chosen": 83.61566162109375,
+      "logits/rejected": 83.63754272460938,
+      "logps/chosen": -33.954978942871094,
+      "logps/rejected": -31.89337158203125,
+      "loss": 0.8073,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.21000584959983826,
+      "rewards/margins": 0.29570215940475464,
+      "rewards/rejected": -0.08569628745317459,
       "step": 100
     },
     {
       "epoch": 0.26,
-      "eval_logits/chosen": 98.9691390991211,
-      "eval_logits/rejected": 98.96244812011719,
-      "eval_logps/chosen": -32.53602981567383,
-      "eval_logps/rejected": -35.98977279663086,
-      "eval_loss": 1.7336534261703491,
-      "eval_rewards/accuracies": 0.4746677577495575,
-      "eval_rewards/chosen": -0.08356913179159164,
-      "eval_rewards/margins": -0.06264925748109818,
-      "eval_rewards/rejected": -0.02091986872255802,
-      "eval_runtime": 104.1909,
-      "eval_samples_per_second": 3.292,
-      "eval_steps_per_second": 0.413,
+      "eval_logits/chosen": 98.75115966796875,
+      "eval_logits/rejected": 98.7406005859375,
+      "eval_logps/chosen": -32.40181350708008,
+      "eval_logps/rejected": -35.920406341552734,
+      "eval_loss": 1.0338481664657593,
+      "eval_rewards/accuracies": 0.4966777563095093,
+      "eval_rewards/chosen": 0.02481812983751297,
+      "eval_rewards/margins": -0.002855697413906455,
+      "eval_rewards/rejected": 0.02767382375895977,
+      "eval_runtime": 104.4142,
+      "eval_samples_per_second": 3.285,
+      "eval_steps_per_second": 0.412,
       "step": 100
     },
     {
       "epoch": 0.29,
-      "grad_norm": 256.0,
+      "grad_norm": 79.0,
       "learning_rate": 4.498257201263691e-06,
-      "logits/chosen": 84.40672302246094,
-      "logits/rejected": 84.30145263671875,
-      "logps/chosen": -32.60901641845703,
-      "logps/rejected": -32.70885467529297,
-      "loss": 3.1161,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.37241029739379883,
-      "rewards/margins": 0.4200804829597473,
-      "rewards/rejected": -0.04767021909356117,
+      "logits/chosen": 83.83598327636719,
+      "logits/rejected": 83.71295928955078,
+      "logps/chosen": -32.41645431518555,
+      "logps/rejected": -32.721168518066406,
+      "loss": 0.8304,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.3638074994087219,
+      "rewards/margins": 0.4029727876186371,
+      "rewards/rejected": -0.03916532173752785,
       "step": 110
     },
     {
       "epoch": 0.31,
-      "grad_norm": 446.0,
+      "grad_norm": 80.5,
       "learning_rate": 4.353806263777678e-06,
-      "logits/chosen": 84.58103942871094,
-      "logits/rejected": 84.6691665649414,
-      "logps/chosen": -28.995365142822266,
-      "logps/rejected": -35.473243713378906,
-      "loss": 2.9665,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": -0.05920146033167839,
-      "rewards/margins": -0.0465129017829895,
-      "rewards/rejected": -0.012688541784882545,
+      "logits/chosen": 83.92488098144531,
+      "logits/rejected": 84.05035400390625,
+      "logps/chosen": -28.292224884033203,
+      "logps/rejected": -35.42224884033203,
+      "loss": 0.7915,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.3824160099029541,
+      "rewards/margins": 0.36027655005455017,
+      "rewards/rejected": 0.022139430046081543,
       "step": 120
     },
     {
       "epoch": 0.34,
-      "grad_norm": 208.0,
+      "grad_norm": 48.5,
       "learning_rate": 4.1940827077152755e-06,
-      "logits/chosen": 82.01698303222656,
-      "logits/rejected": 82.04023742675781,
-      "logps/chosen": -30.6518497467041,
-      "logps/rejected": -32.01683807373047,
-      "loss": 1.858,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.22995714843273163,
-      "rewards/margins": 0.3066490590572357,
-      "rewards/rejected": -0.0766918882727623,
+      "logits/chosen": 81.21788024902344,
+      "logits/rejected": 81.23727416992188,
+      "logps/chosen": -30.410778045654297,
+      "logps/rejected": -32.07198715209961,
+      "loss": 0.828,
+      "rewards/accuracies": 0.612500011920929,
+      "rewards/chosen": 0.2979464530944824,
+      "rewards/margins": 0.38216376304626465,
+      "rewards/rejected": -0.08421732485294342,
       "step": 130
     },
     {
       "epoch": 0.36,
-      "grad_norm": 392.0,
+      "grad_norm": 62.5,
       "learning_rate": 4.0204024186666215e-06,
-      "logits/chosen": 83.59390258789062,
-      "logits/rejected": 83.58688354492188,
-      "logps/chosen": -27.2325439453125,
-      "logps/rejected": -32.482357025146484,
-      "loss": 2.0491,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.19329962134361267,
-      "rewards/margins": 0.24638020992279053,
-      "rewards/rejected": -0.05308058112859726,
+      "logits/chosen": 82.4339828491211,
+      "logits/rejected": 82.45420837402344,
+      "logps/chosen": -27.09214210510254,
+      "logps/rejected": -33.011024475097656,
+      "loss": 0.6935,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.213111013174057,
+      "rewards/margins": 0.5656993985176086,
+      "rewards/rejected": -0.35258835554122925,
       "step": 140
     },
     {
       "epoch": 0.39,
-      "grad_norm": 209.0,
+      "grad_norm": 54.75,
       "learning_rate": 3.834196265035119e-06,
-      "logits/chosen": 82.43211364746094,
-      "logits/rejected": 82.41563415527344,
-      "logps/chosen": -28.861217498779297,
-      "logps/rejected": -32.51410675048828,
-      "loss": 1.804,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.4751041829586029,
-      "rewards/margins": 0.42794743180274963,
-      "rewards/rejected": 0.04715672880411148,
+      "logits/chosen": 81.10945129394531,
+      "logits/rejected": 81.082763671875,
+      "logps/chosen": -28.871135711669922,
+      "logps/rejected": -32.9850959777832,
+      "loss": 0.6411,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.31078284978866577,
+      "rewards/margins": 0.5619392395019531,
+      "rewards/rejected": -0.25115638971328735,
       "step": 150
     },
     {
       "epoch": 0.42,
-      "grad_norm": 450.0,
+      "grad_norm": 63.25,
       "learning_rate": 3.636998309800573e-06,
-      "logits/chosen": 84.4583969116211,
-      "logits/rejected": 84.46671295166016,
-      "logps/chosen": -33.707855224609375,
-      "logps/rejected": -29.813602447509766,
-      "loss": 3.5567,
-      "rewards/accuracies": 0.5375000238418579,
-      "rewards/chosen": 0.38600218296051025,
-      "rewards/margins": 0.2609595060348511,
-      "rewards/rejected": 0.12504267692565918,
+      "logits/chosen": 83.00135803222656,
+      "logits/rejected": 82.99880981445312,
+      "logps/chosen": -33.499019622802734,
+      "logps/rejected": -30.40065574645996,
+      "loss": 0.6874,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.3826376497745514,
+      "rewards/margins": 0.6515066027641296,
+      "rewards/rejected": -0.26886898279190063,
       "step": 160
     },
     {
       "epoch": 0.44,
-      "grad_norm": 278.0,
+      "grad_norm": 68.0,
       "learning_rate": 3.4304331721118078e-06,
-      "logits/chosen": 85.22765350341797,
-      "logits/rejected": 85.16728973388672,
-      "logps/chosen": -31.05304527282715,
-      "logps/rejected": -32.11688995361328,
-      "loss": 1.9364,
-      "rewards/accuracies": 0.6000000238418579,
-      "rewards/chosen": 0.1856154203414917,
-      "rewards/margins": 0.2727610170841217,
-      "rewards/rejected": -0.08714555948972702,
+      "logits/chosen": 83.82991790771484,
+      "logits/rejected": 83.77062225341797,
+      "logps/chosen": -30.814950942993164,
+      "logps/rejected": -32.44989776611328,
+      "loss": 0.7545,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.2666035592556,
+      "rewards/margins": 0.5245082974433899,
+      "rewards/rejected": -0.2579047381877899,
       "step": 170
     },
     {
       "epoch": 0.47,
-      "grad_norm": 196.0,
+      "grad_norm": 50.25,
       "learning_rate": 3.2162026428305436e-06,
-      "logits/chosen": 82.93448638916016,
-      "logits/rejected": 82.9182357788086,
-      "logps/chosen": -30.893056869506836,
-      "logps/rejected": -31.30733299255371,
-      "loss": 1.6619,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.2736847400665283,
-      "rewards/margins": 0.40879687666893005,
-      "rewards/rejected": -0.13511209189891815,
+      "logits/chosen": 81.48609924316406,
+      "logits/rejected": 81.4673843383789,
+      "logps/chosen": -30.380239486694336,
+      "logps/rejected": -31.43511962890625,
+      "loss": 0.6492,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.49014702439308167,
+      "rewards/margins": 0.6568924188613892,
+      "rewards/rejected": -0.16674548387527466,
       "step": 180
     },
     {
       "epoch": 0.49,
-      "grad_norm": 130.0,
+      "grad_norm": 33.5,
       "learning_rate": 2.996071664294641e-06,
-      "logits/chosen": 84.62263488769531,
-      "logits/rejected": 84.59950256347656,
-      "logps/chosen": -30.79315757751465,
-      "logps/rejected": -30.666366577148438,
-      "loss": 2.2923,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": -0.02406691387295723,
-      "rewards/margins": 0.15444278717041016,
-      "rewards/rejected": -0.17850971221923828,
+      "logits/chosen": 83.15840148925781,
+      "logits/rejected": 83.14761352539062,
+      "logps/chosen": -30.205703735351562,
+      "logps/rejected": -30.463912963867188,
+      "loss": 0.9036,
+      "rewards/accuracies": 0.6000000238418579,
+      "rewards/chosen": 0.33642834424972534,
+      "rewards/margins": 0.33396121859550476,
+      "rewards/rejected": 0.0024670884013175964,
       "step": 190
     },
     {
       "epoch": 0.52,
-      "grad_norm": 536.0,
+      "grad_norm": 54.75,
       "learning_rate": 2.7718537898066833e-06,
-      "logits/chosen": 79.89015197753906,
-      "logits/rejected": 79.83697509765625,
-      "logps/chosen": -34.30753707885742,
-      "logps/rejected": -32.36396026611328,
-      "loss": 3.1221,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.40067988634109497,
-      "rewards/margins": 0.2841777205467224,
-      "rewards/rejected": 0.11650214344263077,
+      "logits/chosen": 78.7307357788086,
+      "logits/rejected": 78.67295837402344,
+      "logps/chosen": -33.62441635131836,
+      "logps/rejected": -32.52568054199219,
+      "loss": 0.7227,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.6769934892654419,
+      "rewards/margins": 0.6963596343994141,
+      "rewards/rejected": -0.019366104155778885,
       "step": 200
     },
     {
       "epoch": 0.52,
-      "eval_logits/chosen": 99.16912841796875,
-      "eval_logits/rejected": 99.15798950195312,
-      "eval_logps/chosen": -32.52165603637695,
-      "eval_logps/rejected": -36.10483932495117,
-      "eval_loss": 1.716450572013855,
-      "eval_rewards/accuracies": 0.5278239250183105,
-      "eval_rewards/chosen": -0.0706300213932991,
-      "eval_rewards/margins": 0.05385042726993561,
-      "eval_rewards/rejected": -0.12448045611381531,
-      "eval_runtime": 103.9441,
-      "eval_samples_per_second": 3.3,
-      "eval_steps_per_second": 0.414,
+      "eval_logits/chosen": 98.77264404296875,
+      "eval_logits/rejected": 98.75276947021484,
+      "eval_logps/chosen": -32.62995529174805,
+      "eval_logps/rejected": -36.19297790527344,
+      "eval_loss": 1.0227326154708862,
+      "eval_rewards/accuracies": 0.49501657485961914,
+      "eval_rewards/chosen": -0.11206817626953125,
+      "eval_rewards/margins": 0.023798702284693718,
+      "eval_rewards/rejected": -0.13586686551570892,
+      "eval_runtime": 104.1915,
+      "eval_samples_per_second": 3.292,
+      "eval_steps_per_second": 0.413,
       "step": 200
     },
     {
       "epoch": 0.55,
-      "grad_norm": 392.0,
+      "grad_norm": 65.5,
       "learning_rate": 2.5453962426402006e-06,
-      "logits/chosen": 82.42179870605469,
-      "logits/rejected": 82.31734466552734,
-      "logps/chosen": -33.38166427612305,
-      "logps/rejected": -35.02565383911133,
-      "loss": 2.7032,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.507649302482605,
-      "rewards/margins": 0.5432528257369995,
-      "rewards/rejected": -0.03560344874858856,
+      "logits/chosen": 81.24237823486328,
+      "logits/rejected": 81.15150451660156,
+      "logps/chosen": -33.17626953125,
+      "logps/rejected": -35.2652587890625,
+      "loss": 0.6447,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.46167102456092834,
+      "rewards/margins": 0.6291700601577759,
+      "rewards/rejected": -0.1674990952014923,
       "step": 210
     },
     {
       "epoch": 0.57,
-      "grad_norm": 474.0,
+      "grad_norm": 48.5,
       "learning_rate": 2.3185646976551794e-06,
-      "logits/chosen": 84.43553161621094,
-      "logits/rejected": 84.52201080322266,
-      "logps/chosen": -31.842187881469727,
-      "logps/rejected": -30.83688735961914,
-      "loss": 3.6089,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.09608902782201767,
-      "rewards/margins": 0.1932254582643509,
-      "rewards/rejected": -0.09713643789291382,
+      "logits/chosen": 83.28003692626953,
+      "logits/rejected": 83.37066650390625,
+      "logps/chosen": -31.01590347290039,
+      "logps/rejected": -31.09220314025879,
+      "loss": 0.5863,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.5598315000534058,
+      "rewards/margins": 0.7777799367904663,
+      "rewards/rejected": -0.21794843673706055,
       "step": 220
     },
     {
       "epoch": 0.6,
-      "grad_norm": 324.0,
+      "grad_norm": 62.75,
       "learning_rate": 2.0932279108998323e-06,
-      "logits/chosen": 81.6541976928711,
-      "logits/rejected": 81.72249603271484,
-      "logps/chosen": -32.656639099121094,
-      "logps/rejected": -33.97739791870117,
-      "loss": 1.7997,
-      "rewards/accuracies": 0.5625,
-      "rewards/chosen": 0.1341962367296219,
-      "rewards/margins": 0.10292205959558487,
-      "rewards/rejected": 0.03127415105700493,
+      "logits/chosen": 80.46641540527344,
+      "logits/rejected": 80.52613067626953,
+      "logps/chosen": -32.29460525512695,
+      "logps/rejected": -34.29331588745117,
+      "loss": 0.8345,
+      "rewards/accuracies": 0.637499988079071,
+      "rewards/chosen": 0.30668336153030396,
+      "rewards/margins": 0.4753844141960144,
+      "rewards/rejected": -0.16870108246803284,
       "step": 230
     },
     {
       "epoch": 0.62,
-      "grad_norm": 328.0,
+      "grad_norm": 74.0,
       "learning_rate": 1.8712423238279358e-06,
-      "logits/chosen": 83.9974365234375,
-      "logits/rejected": 84.2688217163086,
-      "logps/chosen": -31.19537353515625,
-      "logps/rejected": -31.57509994506836,
-      "loss": 2.2222,
-      "rewards/accuracies": 0.6499999761581421,
-      "rewards/chosen": 0.443420946598053,
-      "rewards/margins": 0.4821735918521881,
-      "rewards/rejected": -0.038752567023038864,
+      "logits/chosen": 82.80165100097656,
+      "logits/rejected": 83.07733917236328,
+      "logps/chosen": -30.73067283630371,
+      "logps/rejected": -31.75331687927246,
+      "loss": 0.5911,
+      "rewards/accuracies": 0.762499988079071,
+      "rewards/chosen": 0.5744358897209167,
+      "rewards/margins": 0.7072005271911621,
+      "rewards/rejected": -0.13276457786560059,
       "step": 240
     },
     {
       "epoch": 0.65,
-      "grad_norm": 404.0,
+      "grad_norm": 69.0,
       "learning_rate": 1.6544367689701824e-06,
-      "logits/chosen": 82.76817321777344,
-      "logits/rejected": 82.85389709472656,
-      "logps/chosen": -27.292465209960938,
-      "logps/rejected": -29.717370986938477,
-      "loss": 2.0726,
-      "rewards/accuracies": 0.48750001192092896,
-      "rewards/chosen": 0.27308744192123413,
-      "rewards/margins": 0.10805711895227432,
-      "rewards/rejected": 0.1650303304195404,
+      "logits/chosen": 81.53695678710938,
+      "logits/rejected": 81.60115051269531,
+      "logps/chosen": -26.971792221069336,
+      "logps/rejected": -30.23346519470215,
+      "loss": 0.7449,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.3744624853134155,
+      "rewards/margins": 0.5740996599197388,
+      "rewards/rejected": -0.19963720440864563,
       "step": 250
     },
     {
       "epoch": 0.68,
-      "grad_norm": 672.0,
+      "grad_norm": 45.5,
       "learning_rate": 1.4445974030621963e-06,
-      "logits/chosen": 79.98811340332031,
-      "logits/rejected": 80.15504455566406,
-      "logps/chosen": -31.048583984375,
-      "logps/rejected": -35.97705841064453,
-      "loss": 3.1081,
-      "rewards/accuracies": 0.550000011920929,
-      "rewards/chosen": 0.4611719250679016,
-      "rewards/margins": 0.3490581512451172,
-      "rewards/rejected": 0.112113818526268,
+      "logits/chosen": 78.7826156616211,
+      "logits/rejected": 78.90965270996094,
+      "logps/chosen": -30.57162857055664,
+      "logps/rejected": -36.598594665527344,
+      "loss": 0.5112,
+      "rewards/accuracies": 0.7749999761581421,
+      "rewards/chosen": 0.5936219692230225,
+      "rewards/margins": 0.8918012380599976,
+      "rewards/rejected": -0.29817938804626465,
       "step": 260
     },
     {
       "epoch": 0.7,
-      "grad_norm": 211.0,
+      "grad_norm": 38.25,
       "learning_rate": 1.243452991757889e-06,
-      "logits/chosen": 79.3856201171875,
-      "logits/rejected": 79.41062927246094,
-      "logps/chosen": -31.38250732421875,
-      "logps/rejected": -31.722015380859375,
-      "loss": 2.0801,
-      "rewards/accuracies": 0.637499988079071,
-      "rewards/chosen": 0.3567788600921631,
-      "rewards/margins": 0.5517110228538513,
-      "rewards/rejected": -0.194932222366333,
+      "logits/chosen": 78.14024353027344,
+      "logits/rejected": 78.16914367675781,
+      "logps/chosen": -30.831195831298828,
+      "logps/rejected": -31.901092529296875,
+      "loss": 0.5884,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.56863933801651,
+      "rewards/margins": 0.8060423135757446,
+      "rewards/rejected": -0.23740296065807343,
       "step": 270
     },
     {
       "epoch": 0.73,
-      "grad_norm": 498.0,
+      "grad_norm": 65.0,
       "learning_rate": 1.0526606671603523e-06,
-      "logits/chosen": 81.8171157836914,
-      "logits/rejected": 81.62063598632812,
-      "logps/chosen": -31.32455062866211,
-      "logps/rejected": -29.448989868164062,
-      "loss": 2.1832,
-      "rewards/accuracies": 0.612500011920929,
-      "rewards/chosen": 0.43680915236473083,
-      "rewards/margins": 0.2406042516231537,
-      "rewards/rejected": 0.19620487093925476,
+      "logits/chosen": 80.71138000488281,
+      "logits/rejected": 80.48588562011719,
+      "logps/chosen": -31.13303565979004,
+      "logps/rejected": -29.891042709350586,
+      "loss": 0.751,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.4061153829097748,
+      "rewards/margins": 0.540542483329773,
+      "rewards/rejected": -0.13442707061767578,
       "step": 280
     },
     {
       "epoch": 0.75,
-      "grad_norm": 262.0,
+      "grad_norm": 47.75,
       "learning_rate": 8.737922755071455e-07,
-      "logits/chosen": 82.0579833984375,
-      "logits/rejected": 81.96476745605469,
-      "logps/chosen": -33.8171272277832,
-      "logps/rejected": -31.962472915649414,
-      "loss": 3.4215,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.25841274857521057,
-      "rewards/margins": 0.2094600647687912,
-      "rewards/rejected": 0.04895265772938728,
+      "logits/chosen": 80.93512725830078,
+      "logits/rejected": 80.84117126464844,
+      "logps/chosen": -32.98380661010742,
+      "logps/rejected": -32.5334587097168,
+      "loss": 0.4856,
+      "rewards/accuracies": 0.7875000238418579,
+      "rewards/chosen": 0.672263503074646,
+      "rewards/margins": 0.9822207689285278,
+      "rewards/rejected": -0.30995720624923706,
       "step": 290
     },
     {
       "epoch": 0.78,
-      "grad_norm": 169.0,
+      "grad_norm": 51.75,
       "learning_rate": 7.08321427484816e-07,
-      "logits/chosen": 77.75779724121094,
-      "logits/rejected": 77.80491638183594,
-      "logps/chosen": -32.91708755493164,
-      "logps/rejected": -29.0297908782959,
-      "loss": 2.4404,
-      "rewards/accuracies": 0.675000011920929,
-      "rewards/chosen": 0.4397595524787903,
-      "rewards/margins": 0.37474992871284485,
-      "rewards/rejected": 0.06500961631536484,
+      "logits/chosen": 76.60159301757812,
+      "logits/rejected": 76.69679260253906,
+      "logps/chosen": -31.996997833251953,
+      "logps/rejected": -29.191226959228516,
+      "loss": 0.6355,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.8452256321907043,
+      "rewards/margins": 0.8987467885017395,
+      "rewards/rejected": -0.05352109670639038,
       "step": 300
     },
     {
       "epoch": 0.78,
-      "eval_logits/chosen": 99.06168365478516,
-      "eval_logits/rejected": 99.05675506591797,
-      "eval_logps/chosen": -32.30947494506836,
-      "eval_logps/rejected": -35.82986068725586,
-      "eval_loss": 1.5173850059509277,
-      "eval_rewards/accuracies": 0.510797381401062,
-      "eval_rewards/chosen": 0.12033051997423172,
-      "eval_rewards/margins": -0.0026713553816080093,
-      "eval_rewards/rejected": 0.12300187349319458,
-      "eval_runtime": 103.9278,
-      "eval_samples_per_second": 3.3,
+      "eval_logits/chosen": 98.76406860351562,
+      "eval_logits/rejected": 98.7339859008789,
+      "eval_logps/chosen": -32.608062744140625,
+      "eval_logps/rejected": -36.16687774658203,
+      "eval_loss": 1.0328983068466187,
+      "eval_rewards/accuracies": 0.5456810593605042,
+      "eval_rewards/chosen": -0.09893348813056946,
+      "eval_rewards/margins": 0.02127554826438427,
+      "eval_rewards/rejected": -0.12020901590585709,
+      "eval_runtime": 103.9872,
+      "eval_samples_per_second": 3.298,
       "eval_steps_per_second": 0.414,
       "step": 300
     },
     {
       "epoch": 0.81,
-      "grad_norm": 928.0,
+      "grad_norm": 53.25,
       "learning_rate": 5.576113578589035e-07,
-      "logits/chosen": 84.6921157836914,
-      "logits/rejected": 84.73957824707031,
-      "logps/chosen": -30.482025146484375,
-      "logps/rejected": -32.063751220703125,
-      "loss": 2.7405,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.3433563709259033,
-      "rewards/margins": 0.22095146775245667,
-      "rewards/rejected": 0.12240489572286606,
+      "logits/chosen": 83.55947875976562,
+      "logits/rejected": 83.5940170288086,
+      "logps/chosen": -29.987110137939453,
+      "logps/rejected": -32.35546112060547,
+      "loss": 0.6477,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.5258544683456421,
+      "rewards/margins": 0.6192765831947327,
+      "rewards/rejected": -0.09342211484909058,
       "step": 310
     },
     {
       "epoch": 0.83,
-      "grad_norm": 406.0,
+      "grad_norm": 46.5,
       "learning_rate": 4.229036944380913e-07,
-      "logits/chosen": 82.20662689208984,
-      "logits/rejected": 82.20085906982422,
-      "logps/chosen": -31.305639266967773,
-      "logps/rejected": -28.786062240600586,
-      "loss": 2.2147,
-      "rewards/accuracies": 0.4749999940395355,
-      "rewards/chosen": 0.2636275887489319,
-      "rewards/margins": 0.14354394376277924,
-      "rewards/rejected": 0.12008367478847504,
+      "logits/chosen": 81.14942932128906,
+      "logits/rejected": 81.15238952636719,
+      "logps/chosen": -30.662851333618164,
+      "logps/rejected": -29.089075088500977,
+      "loss": 0.6327,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": 0.5614250898361206,
+      "rewards/margins": 0.6631743311882019,
+      "rewards/rejected": -0.10174927860498428,
       "step": 320
     },
     {
       "epoch": 0.86,
-      "grad_norm": 366.0,
+      "grad_norm": 43.5,
       "learning_rate": 3.053082288996112e-07,
-      "logits/chosen": 79.59921264648438,
-      "logits/rejected": 79.63972473144531,
-      "logps/chosen": -29.793676376342773,
-      "logps/rejected": -32.53992462158203,
-      "loss": 2.2293,
-      "rewards/accuracies": 0.625,
-      "rewards/chosen": 0.5030375719070435,
-      "rewards/margins": 0.41758814454078674,
-      "rewards/rejected": 0.08544941991567612,
+      "logits/chosen": 78.36723327636719,
+      "logits/rejected": 78.41484832763672,
+      "logps/chosen": -29.124780654907227,
+      "logps/rejected": -32.84328079223633,
+      "loss": 0.6057,
+      "rewards/accuracies": 0.675000011920929,
+      "rewards/chosen": 0.7366967797279358,
+      "rewards/margins": 0.8617448806762695,
+      "rewards/rejected": -0.12504807114601135,
       "step": 330
     },
     {
       "epoch": 0.88,
-      "grad_norm": 300.0,
+      "grad_norm": 65.5,
       "learning_rate": 2.0579377374915805e-07,
-      "logits/chosen": 83.62577056884766,
-      "logits/rejected": 83.62586975097656,
-      "logps/chosen": -32.759605407714844,
-      "logps/rejected": -33.222259521484375,
-      "loss": 2.4595,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.40729111433029175,
-      "rewards/margins": 0.2755037844181061,
-      "rewards/rejected": 0.13178732991218567,
+      "logits/chosen": 82.55430603027344,
+      "logits/rejected": 82.59879302978516,
+      "logps/chosen": -32.09717559814453,
+      "logps/rejected": -33.71881103515625,
+      "loss": 0.6103,
+      "rewards/accuracies": 0.699999988079071,
+      "rewards/chosen": 0.6689842343330383,
+      "rewards/margins": 0.8790559768676758,
+      "rewards/rejected": -0.2100718766450882,
       "step": 340
     },
     {
       "epoch": 0.91,
-      "grad_norm": 302.0,
+      "grad_norm": 45.25,
       "learning_rate": 1.2518018074041684e-07,
-      "logits/chosen": 82.8216781616211,
-      "logits/rejected": 82.81562805175781,
-      "logps/chosen": -33.33207702636719,
-      "logps/rejected": -32.93452835083008,
-      "loss": 1.9575,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.2804979979991913,
-      "rewards/margins": 0.13895203173160553,
-      "rewards/rejected": 0.14154598116874695,
+      "logits/chosen": 81.6875228881836,
+      "logits/rejected": 81.70503234863281,
+      "logps/chosen": -32.540645599365234,
+      "logps/rejected": -33.16196823120117,
+      "loss": 0.6663,
+      "rewards/accuracies": 0.7250000238418579,
+      "rewards/chosen": 0.6618569493293762,
+      "rewards/margins": 0.7039581537246704,
+      "rewards/rejected": -0.042101241648197174,
       "step": 350
     },
     {
       "epoch": 0.94,
-      "grad_norm": 218.0,
+      "grad_norm": 37.5,
       "learning_rate": 6.41315865106129e-08,
-      "logits/chosen": 84.19699096679688,
-      "logits/rejected": 84.23506164550781,
-      "logps/chosen": -28.99674415588379,
-      "logps/rejected": -31.437408447265625,
-      "loss": 2.09,
-      "rewards/accuracies": 0.574999988079071,
-      "rewards/chosen": 0.4999212324619293,
-      "rewards/margins": 0.3087221086025238,
-      "rewards/rejected": 0.19119907915592194,
+      "logits/chosen": 83.08977508544922,
+      "logits/rejected": 83.10887145996094,
+      "logps/chosen": -28.5622615814209,
+      "logps/rejected": -31.776697158813477,
+      "loss": 0.6199,
+      "rewards/accuracies": 0.737500011920929,
+      "rewards/chosen": 0.5939711332321167,
+      "rewards/margins": 0.6700795292854309,
+      "rewards/rejected": -0.07610837370157242,
       "step": 360
     },
     {
       "epoch": 0.96,
-      "grad_norm": 508.0,
+      "grad_norm": 69.0,
       "learning_rate": 2.3150941078050325e-08,
-      "logits/chosen": 83.60005187988281,
-      "logits/rejected": 83.62088012695312,
-      "logps/chosen": -32.41377258300781,
-      "logps/rejected": -34.801570892333984,
-      "loss": 3.3754,
-      "rewards/accuracies": 0.5,
-      "rewards/chosen": 0.41615089774131775,
-      "rewards/margins": 0.13912765681743622,
-      "rewards/rejected": 0.2770232558250427,
+      "logits/chosen": 82.51141357421875,
+      "logits/rejected": 82.53152465820312,
+      "logps/chosen": -31.914348602294922,
+      "logps/rejected": -35.504486083984375,
+      "loss": 0.6338,
+      "rewards/accuracies": 0.6499999761581421,
+      "rewards/chosen": 0.5770877599716187,
+      "rewards/margins": 0.8141579627990723,
+      "rewards/rejected": -0.237070232629776,
       "step": 370
     },
     {
       "epoch": 0.99,
-      "grad_norm": 194.0,
+      "grad_norm": 52.75,
       "learning_rate": 2.575864278703266e-09,
-      "logits/chosen": 77.62187194824219,
-      "logits/rejected": 77.48942565917969,
-      "logps/chosen": -30.352733612060547,
-      "logps/rejected": -28.094280242919922,
-      "loss": 2.3864,
-      "rewards/accuracies": 0.512499988079071,
-      "rewards/chosen": 0.20105357468128204,
-      "rewards/margins": 0.14138910174369812,
-      "rewards/rejected": 0.059664465487003326,
+      "logits/chosen": 76.49024963378906,
+      "logits/rejected": 76.35295104980469,
+      "logps/chosen": -29.67061996459961,
+      "logps/rejected": -28.207141876220703,
+      "loss": 0.6675,
+      "rewards/accuracies": 0.7124999761581421,
+      "rewards/chosen": 0.5433036088943481,
+      "rewards/margins": 0.571243166923523,
+      "rewards/rejected": -0.027939552441239357,
       "step": 380
     },
     {
       "epoch": 1.0,
       "step": 385,
       "total_flos": 0.0,
-      "train_loss": 2.2499145250041765,
-      "train_runtime": 2558.7273,
-      "train_samples_per_second": 1.203,
-      "train_steps_per_second": 0.15
+      "train_loss": 0.746250953302755,
+      "train_runtime": 2556.1232,
+      "train_samples_per_second": 1.205,
+      "train_steps_per_second": 0.151
     }
   ],
   "logging_steps": 10,