hugodk-sch commited on
Commit
32c19f6
1 Parent(s): 228bd09

Model save

Browse files
Files changed (5) hide show
  1. README.md +13 -16
  2. adapter_model.safetensors +1 -1
  3. all_results.json +3 -16
  4. train_results.json +3 -3
  5. trainer_state.json +376 -376
README.md CHANGED
@@ -1,13 +1,10 @@
1
  ---
2
  library_name: peft
3
  tags:
4
- - alignment-handbook
5
  - trl
6
  - dpo
7
  - generated_from_trainer
8
  base_model: NbAiLab/nb-gpt-j-6B-v2
9
- datasets:
10
- - hugodk-sch/aftonposten_title_prefs
11
  model-index:
12
  - name: aftonposten-6b-align-scan
13
  results: []
@@ -18,17 +15,17 @@ should probably proofread and complete it, then remove this comment. -->
18
 
19
  # aftonposten-6b-align-scan
20
 
21
- This model is a fine-tuned version of [data/ap-gpt-j-6b-sft-qlora-04-08](https://huggingface.co/data/ap-gpt-j-6b-sft-qlora-04-08) on the hugodk-sch/aftonposten_title_prefs dataset.
22
  It achieves the following results on the evaluation set:
23
- - Loss: 0.4947
24
- - Rewards/chosen: 0.1656
25
- - Rewards/rejected: 0.1437
26
- - Rewards/accuracies: 0.5365
27
- - Rewards/margins: 0.0219
28
- - Logps/rejected: -37.2771
29
- - Logps/chosen: -33.7585
30
- - Logits/rejected: -2.2259
31
- - Logits/chosen: -2.2307
32
 
33
  ## Model description
34
 
@@ -63,9 +60,9 @@ The following hyperparameters were used during training:
63
 
64
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
65
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
66
- | 0.4799 | 0.26 | 100 | 0.4987 | 0.0790 | 0.0734 | 0.5104 | 0.0056 | -37.3943 | -33.9029 | -2.2301 | -2.2349 |
67
- | 0.4548 | 0.52 | 200 | 0.4956 | 0.1590 | 0.1392 | 0.5341 | 0.0198 | -37.2846 | -33.7696 | -2.2287 | -2.2335 |
68
- | 0.41 | 0.78 | 300 | 0.4937 | 0.1639 | 0.1391 | 0.5361 | 0.0248 | -37.2848 | -33.7614 | -2.2261 | -2.2309 |
69
 
70
 
71
  ### Framework versions
 
1
  ---
2
  library_name: peft
3
  tags:
 
4
  - trl
5
  - dpo
6
  - generated_from_trainer
7
  base_model: NbAiLab/nb-gpt-j-6B-v2
 
 
8
  model-index:
9
  - name: aftonposten-6b-align-scan
10
  results: []
 
15
 
16
  # aftonposten-6b-align-scan
17
 
18
+ This model is a fine-tuned version of [NbAiLab/nb-gpt-j-6B-v2](https://huggingface.co/NbAiLab/nb-gpt-j-6B-v2) on an unknown dataset.
19
  It achieves the following results on the evaluation set:
20
+ - Loss: 0.7655
21
+ - Rewards/chosen: -0.0166
22
+ - Rewards/rejected: -0.0295
23
+ - Rewards/accuracies: 0.5071
24
+ - Rewards/margins: 0.0129
25
+ - Logps/rejected: -37.5657
26
+ - Logps/chosen: -34.0622
27
+ - Logits/rejected: -2.2285
28
+ - Logits/chosen: -2.2333
29
 
30
  ## Model description
31
 
 
60
 
61
  | Training Loss | Epoch | Step | Validation Loss | Rewards/chosen | Rewards/rejected | Rewards/accuracies | Rewards/margins | Logps/rejected | Logps/chosen | Logits/rejected | Logits/chosen |
62
  |:-------------:|:-----:|:----:|:---------------:|:--------------:|:----------------:|:------------------:|:---------------:|:--------------:|:------------:|:---------------:|:-------------:|
63
+ | 0.5903 | 0.26 | 100 | 0.7450 | 0.0123 | 0.0047 | 0.5241 | 0.0075 | -37.5087 | -34.0141 | -2.2318 | -2.2366 |
64
+ | 0.59 | 0.52 | 200 | 0.7500 | -0.0208 | -0.0352 | 0.5365 | 0.0144 | -37.5752 | -34.0691 | -2.2284 | -2.2332 |
65
+ | 0.3871 | 0.78 | 300 | 0.7655 | -0.0166 | -0.0295 | 0.5071 | 0.0129 | -37.5657 | -34.0622 | -2.2285 | -2.2333 |
66
 
67
 
68
  ### Framework versions
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:29d81ccb9aee98dd50644ca8a0d7a00dd0b18fd386da537748fc75ae8a58b388
3
  size 176183216
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:61fdd9123d70c8ba17ec6ad03dd0e97d84b7e40a5f415a1d5e1a038d4ac398a0
3
  size 176183216
all_results.json CHANGED
@@ -1,21 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "eval_logits/chosen": -2.230694055557251,
4
- "eval_logits/rejected": -2.225895643234253,
5
- "eval_logps/chosen": -33.75852966308594,
6
- "eval_logps/rejected": -37.27712631225586,
7
- "eval_loss": 0.4946732223033905,
8
- "eval_rewards/accuracies": 0.5365448594093323,
9
- "eval_rewards/chosen": 0.16561271250247955,
10
- "eval_rewards/margins": 0.02191758342087269,
11
- "eval_rewards/rejected": 0.14369513094425201,
12
- "eval_runtime": 145.7007,
13
- "eval_samples": 343,
14
- "eval_samples_per_second": 2.354,
15
- "eval_steps_per_second": 0.295,
16
- "train_loss": 0.46439293204963983,
17
- "train_runtime": 3253.2697,
18
  "train_samples": 3079,
19
- "train_samples_per_second": 0.946,
20
  "train_steps_per_second": 0.118
21
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 0.5965218513042896,
4
+ "train_runtime": 3252.2803,
 
 
 
 
 
 
 
 
 
 
 
 
 
5
  "train_samples": 3079,
6
+ "train_samples_per_second": 0.947,
7
  "train_steps_per_second": 0.118
8
  }
train_results.json CHANGED
@@ -1,8 +1,8 @@
1
  {
2
  "epoch": 1.0,
3
- "train_loss": 0.46439293204963983,
4
- "train_runtime": 3253.2697,
5
  "train_samples": 3079,
6
- "train_samples_per_second": 0.946,
7
  "train_steps_per_second": 0.118
8
  }
 
1
  {
2
  "epoch": 1.0,
3
+ "train_loss": 0.5965218513042896,
4
+ "train_runtime": 3252.2803,
5
  "train_samples": 3079,
6
+ "train_samples_per_second": 0.947,
7
  "train_steps_per_second": 0.118
8
  }
trainer_state.json CHANGED
@@ -15,7 +15,7 @@
15
  "logits/rejected": -1.7377450466156006,
16
  "logps/chosen": -29.553977966308594,
17
  "logps/rejected": -42.813133239746094,
18
- "loss": 0.5,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
@@ -25,467 +25,467 @@
25
  {
26
  "epoch": 0.03,
27
  "learning_rate": 1.282051282051282e-06,
28
- "logits/chosen": -1.866639494895935,
29
- "logits/rejected": -1.870951533317566,
30
- "logps/chosen": -36.991004943847656,
31
- "logps/rejected": -33.64551544189453,
32
- "loss": 0.4968,
33
- "rewards/accuracies": 0.5694444179534912,
34
- "rewards/chosen": 0.009355764836072922,
35
- "rewards/margins": 0.018620232120156288,
36
- "rewards/rejected": -0.009264465421438217,
37
  "step": 10
38
  },
39
  {
40
  "epoch": 0.05,
41
  "learning_rate": 2.564102564102564e-06,
42
- "logits/chosen": -1.9975944757461548,
43
- "logits/rejected": -2.000239133834839,
44
- "logps/chosen": -29.655420303344727,
45
- "logps/rejected": -29.04937171936035,
46
- "loss": 0.5034,
47
- "rewards/accuracies": 0.44999998807907104,
48
- "rewards/chosen": -0.007936288602650166,
49
- "rewards/margins": -0.016377601772546768,
50
- "rewards/rejected": 0.008441311307251453,
51
  "step": 20
52
  },
53
  {
54
  "epoch": 0.08,
55
  "learning_rate": 3.846153846153847e-06,
56
- "logits/chosen": -1.920345664024353,
57
- "logits/rejected": -1.917649507522583,
58
- "logps/chosen": -31.40158462524414,
59
- "logps/rejected": -33.22117614746094,
60
- "loss": 0.499,
61
  "rewards/accuracies": 0.550000011920929,
62
- "rewards/chosen": 0.008706120774149895,
63
- "rewards/margins": 0.010335424914956093,
64
- "rewards/rejected": -0.0016293060034513474,
65
  "step": 30
66
  },
67
  {
68
  "epoch": 0.1,
69
  "learning_rate": 4.999896948438434e-06,
70
- "logits/chosen": -2.016953706741333,
71
- "logits/rejected": -2.008241891860962,
72
- "logps/chosen": -32.55894088745117,
73
- "logps/rejected": -32.48887634277344,
74
- "loss": 0.501,
75
- "rewards/accuracies": 0.550000011920929,
76
- "rewards/chosen": 0.010718915611505508,
77
- "rewards/margins": -0.0035588580649346113,
78
- "rewards/rejected": 0.014277773909270763,
79
  "step": 40
80
  },
81
  {
82
  "epoch": 0.13,
83
  "learning_rate": 4.987541037542187e-06,
84
- "logits/chosen": -1.8621622323989868,
85
- "logits/rejected": -1.851413369178772,
86
- "logps/chosen": -33.5220832824707,
87
- "logps/rejected": -35.4012565612793,
88
- "loss": 0.5034,
89
- "rewards/accuracies": 0.48750001192092896,
90
- "rewards/chosen": 0.021240826696157455,
91
- "rewards/margins": -0.009411826729774475,
92
- "rewards/rejected": 0.03065265715122223,
93
  "step": 50
94
  },
95
  {
96
  "epoch": 0.16,
97
  "learning_rate": 4.954691471941119e-06,
98
- "logits/chosen": -1.9412288665771484,
99
- "logits/rejected": -1.9431712627410889,
100
- "logps/chosen": -32.506317138671875,
101
- "logps/rejected": -33.202842712402344,
102
- "loss": 0.4846,
103
- "rewards/accuracies": 0.699999988079071,
104
- "rewards/chosen": 0.05631808191537857,
105
- "rewards/margins": 0.07519452273845673,
106
- "rewards/rejected": -0.018876442685723305,
107
  "step": 60
108
  },
109
  {
110
  "epoch": 0.18,
111
  "learning_rate": 4.901618883413549e-06,
112
- "logits/chosen": -2.0726475715637207,
113
- "logits/rejected": -2.077623128890991,
114
- "logps/chosen": -33.90425491333008,
115
- "logps/rejected": -36.56460189819336,
116
- "loss": 0.4903,
117
- "rewards/accuracies": 0.574999988079071,
118
- "rewards/chosen": 0.05054063722491264,
119
- "rewards/margins": 0.04797897860407829,
120
- "rewards/rejected": 0.002561659086495638,
121
  "step": 70
122
  },
123
  {
124
  "epoch": 0.21,
125
  "learning_rate": 4.828760511501322e-06,
126
- "logits/chosen": -1.9338359832763672,
127
- "logits/rejected": -1.9369510412216187,
128
- "logps/chosen": -34.21551513671875,
129
- "logps/rejected": -34.527488708496094,
130
- "loss": 0.4797,
131
- "rewards/accuracies": 0.5874999761581421,
132
- "rewards/chosen": 0.1273767203092575,
133
- "rewards/margins": 0.08842838555574417,
134
- "rewards/rejected": 0.038948334753513336,
135
  "step": 80
136
  },
137
  {
138
  "epoch": 0.23,
139
  "learning_rate": 4.7367166013034295e-06,
140
- "logits/chosen": -1.9420888423919678,
141
- "logits/rejected": -1.9465935230255127,
142
- "logps/chosen": -32.2774772644043,
143
- "logps/rejected": -32.245147705078125,
144
- "loss": 0.4887,
145
- "rewards/accuracies": 0.625,
146
- "rewards/chosen": 0.11396588385105133,
147
- "rewards/margins": 0.04570900648832321,
148
- "rewards/rejected": 0.06825686991214752,
149
  "step": 90
150
  },
151
  {
152
  "epoch": 0.26,
153
  "learning_rate": 4.626245458345211e-06,
154
- "logits/chosen": -2.040524482727051,
155
- "logits/rejected": -2.0385289192199707,
156
- "logps/chosen": -32.03118133544922,
157
- "logps/rejected": -31.205692291259766,
158
- "loss": 0.4799,
159
- "rewards/accuracies": 0.6875,
160
- "rewards/chosen": 0.12579473853111267,
161
- "rewards/margins": 0.08948288857936859,
162
- "rewards/rejected": 0.036311857402324677,
163
  "step": 100
164
  },
165
  {
166
  "epoch": 0.26,
167
- "eval_logits/chosen": -2.234907865524292,
168
- "eval_logits/rejected": -2.2300779819488525,
169
- "eval_logps/chosen": -33.90293884277344,
170
- "eval_logps/rejected": -37.394290924072266,
171
- "eval_loss": 0.4987158477306366,
172
- "eval_rewards/accuracies": 0.5103820562362671,
173
- "eval_rewards/chosen": 0.0789671465754509,
174
- "eval_rewards/margins": 0.005569713190197945,
175
- "eval_rewards/rejected": 0.0733974277973175,
176
- "eval_runtime": 145.9867,
177
- "eval_samples_per_second": 2.35,
178
  "eval_steps_per_second": 0.295,
179
  "step": 100
180
  },
181
  {
182
  "epoch": 0.29,
183
  "learning_rate": 4.498257201263691e-06,
184
- "logits/chosen": -1.9966247081756592,
185
- "logits/rejected": -1.9942585229873657,
186
- "logps/chosen": -33.005943298339844,
187
- "logps/rejected": -33.908599853515625,
188
- "loss": 0.4799,
189
- "rewards/accuracies": 0.6000000238418579,
190
- "rewards/chosen": 0.14272192120552063,
191
- "rewards/margins": 0.07025818526744843,
192
- "rewards/rejected": 0.07246376574039459,
193
  "step": 110
194
  },
195
  {
196
  "epoch": 0.31,
197
  "learning_rate": 4.353806263777678e-06,
198
- "logits/chosen": -2.0064611434936523,
199
- "logits/rejected": -1.9981132745742798,
200
- "logps/chosen": -32.16605758666992,
201
- "logps/rejected": -31.968135833740234,
202
- "loss": 0.4883,
203
- "rewards/accuracies": 0.5,
204
- "rewards/chosen": 0.16730833053588867,
205
- "rewards/margins": 0.0512862391769886,
206
- "rewards/rejected": 0.11602209508419037,
207
  "step": 120
208
  },
209
  {
210
  "epoch": 0.34,
211
  "learning_rate": 4.1940827077152755e-06,
212
- "logits/chosen": -2.0353620052337646,
213
- "logits/rejected": -2.0273985862731934,
214
- "logps/chosen": -30.145355224609375,
215
- "logps/rejected": -31.892078399658203,
216
- "loss": 0.477,
217
- "rewards/accuracies": 0.6000000238418579,
218
- "rewards/chosen": 0.19821622967720032,
219
- "rewards/margins": 0.10165951400995255,
220
- "rewards/rejected": 0.09655670821666718,
221
  "step": 130
222
  },
223
  {
224
  "epoch": 0.36,
225
  "learning_rate": 4.0204024186666215e-06,
226
- "logits/chosen": -1.965121865272522,
227
- "logits/rejected": -1.9753497838974,
228
- "logps/chosen": -31.059722900390625,
229
- "logps/rejected": -32.391807556152344,
230
- "loss": 0.4695,
231
- "rewards/accuracies": 0.6625000238418579,
232
- "rewards/chosen": 0.21596479415893555,
233
- "rewards/margins": 0.13012099266052246,
234
- "rewards/rejected": 0.08584379404783249,
235
  "step": 140
236
  },
237
  {
238
  "epoch": 0.39,
239
  "learning_rate": 3.834196265035119e-06,
240
- "logits/chosen": -1.8772761821746826,
241
- "logits/rejected": -1.8784434795379639,
242
- "logps/chosen": -33.682029724121094,
243
- "logps/rejected": -34.56599044799805,
244
- "loss": 0.4566,
245
- "rewards/accuracies": 0.612500011920929,
246
- "rewards/chosen": 0.30639737844467163,
247
- "rewards/margins": 0.19163966178894043,
248
- "rewards/rejected": 0.11475770175457001,
249
  "step": 150
250
  },
251
  {
252
  "epoch": 0.42,
253
  "learning_rate": 3.636998309800573e-06,
254
- "logits/chosen": -1.9283088445663452,
255
- "logits/rejected": -1.9249660968780518,
256
- "logps/chosen": -35.78284454345703,
257
- "logps/rejected": -32.46989059448242,
258
- "loss": 0.4802,
259
- "rewards/accuracies": 0.6000000238418579,
260
- "rewards/chosen": 0.23330166935920715,
261
- "rewards/margins": 0.08158595860004425,
262
- "rewards/rejected": 0.1517156958580017,
263
  "step": 160
264
  },
265
  {
266
  "epoch": 0.44,
267
  "learning_rate": 3.4304331721118078e-06,
268
- "logits/chosen": -2.0310397148132324,
269
- "logits/rejected": -2.0237185955047607,
270
- "logps/chosen": -33.21892547607422,
271
- "logps/rejected": -31.1767520904541,
272
- "loss": 0.443,
273
- "rewards/accuracies": 0.7124999761581421,
274
- "rewards/chosen": 0.347359299659729,
275
- "rewards/margins": 0.24229975044727325,
276
- "rewards/rejected": 0.10505956411361694,
277
  "step": 170
278
  },
279
  {
280
  "epoch": 0.47,
281
  "learning_rate": 3.2162026428305436e-06,
282
- "logits/chosen": -2.037632703781128,
283
- "logits/rejected": -2.0428080558776855,
284
- "logps/chosen": -31.97260093688965,
285
- "logps/rejected": -32.14643096923828,
286
- "loss": 0.4662,
287
- "rewards/accuracies": 0.6499999761581421,
288
- "rewards/chosen": 0.34331104159355164,
289
- "rewards/margins": 0.1400861144065857,
290
- "rewards/rejected": 0.20322492718696594,
291
  "step": 180
292
  },
293
  {
294
  "epoch": 0.49,
295
  "learning_rate": 2.996071664294641e-06,
296
- "logits/chosen": -2.0373456478118896,
297
- "logits/rejected": -2.0346312522888184,
298
- "logps/chosen": -31.077922821044922,
299
- "logps/rejected": -31.089244842529297,
300
- "loss": 0.4725,
301
- "rewards/accuracies": 0.6000000238418579,
302
- "rewards/chosen": 0.24661760032176971,
303
- "rewards/margins": 0.10923590511083603,
304
- "rewards/rejected": 0.13738170266151428,
305
  "step": 190
306
  },
307
  {
308
  "epoch": 0.52,
309
  "learning_rate": 2.7718537898066833e-06,
310
- "logits/chosen": -1.9075931310653687,
311
- "logits/rejected": -1.9122419357299805,
312
- "logps/chosen": -31.088512420654297,
313
- "logps/rejected": -32.591243743896484,
314
- "loss": 0.4548,
315
- "rewards/accuracies": 0.7250000238418579,
316
- "rewards/chosen": 0.3106623888015747,
317
- "rewards/margins": 0.190850168466568,
318
- "rewards/rejected": 0.11981221288442612,
319
  "step": 200
320
  },
321
  {
322
  "epoch": 0.52,
323
- "eval_logits/chosen": -2.233458995819092,
324
- "eval_logits/rejected": -2.2286527156829834,
325
- "eval_logps/chosen": -33.769569396972656,
326
- "eval_logps/rejected": -37.28458786010742,
327
- "eval_loss": 0.4955619275569916,
328
- "eval_rewards/accuracies": 0.5340532064437866,
329
- "eval_rewards/chosen": 0.15899108350276947,
330
- "eval_rewards/margins": 0.019772635772824287,
331
- "eval_rewards/rejected": 0.13921843469142914,
332
- "eval_runtime": 145.6335,
333
- "eval_samples_per_second": 2.355,
334
  "eval_steps_per_second": 0.295,
335
  "step": 200
336
  },
337
  {
338
  "epoch": 0.55,
339
  "learning_rate": 2.5453962426402006e-06,
340
- "logits/chosen": -2.020599365234375,
341
- "logits/rejected": -2.0312187671661377,
342
- "logps/chosen": -31.51775550842285,
343
- "logps/rejected": -33.68301773071289,
344
- "loss": 0.462,
345
- "rewards/accuracies": 0.637499988079071,
346
- "rewards/chosen": 0.2721289098262787,
347
- "rewards/margins": 0.16431835293769836,
348
- "rewards/rejected": 0.10781057924032211,
349
  "step": 210
350
  },
351
  {
352
  "epoch": 0.57,
353
  "learning_rate": 2.3185646976551794e-06,
354
- "logits/chosen": -1.9131078720092773,
355
- "logits/rejected": -1.9278091192245483,
356
- "logps/chosen": -29.563827514648438,
357
- "logps/rejected": -31.399028778076172,
358
- "loss": 0.4463,
359
- "rewards/accuracies": 0.7124999761581421,
360
- "rewards/chosen": 0.3286263942718506,
361
- "rewards/margins": 0.23313641548156738,
362
- "rewards/rejected": 0.09548995643854141,
363
  "step": 220
364
  },
365
  {
366
  "epoch": 0.6,
367
  "learning_rate": 2.0932279108998323e-06,
368
- "logits/chosen": -1.969555139541626,
369
- "logits/rejected": -1.9735543727874756,
370
- "logps/chosen": -32.82552719116211,
371
- "logps/rejected": -31.39508628845215,
372
- "loss": 0.4396,
373
  "rewards/accuracies": 0.7250000238418579,
374
- "rewards/chosen": 0.3551639914512634,
375
- "rewards/margins": 0.26263612508773804,
376
- "rewards/rejected": 0.0925278514623642,
377
  "step": 230
378
  },
379
  {
380
  "epoch": 0.62,
381
  "learning_rate": 1.8712423238279358e-06,
382
- "logits/chosen": -1.9689340591430664,
383
- "logits/rejected": -1.947165846824646,
384
- "logps/chosen": -33.572689056396484,
385
- "logps/rejected": -34.84184265136719,
386
- "loss": 0.4355,
387
- "rewards/accuracies": 0.6875,
388
- "rewards/chosen": 0.3486354947090149,
389
- "rewards/margins": 0.2832663357257843,
390
- "rewards/rejected": 0.0653691440820694,
391
  "step": 240
392
  },
393
  {
394
  "epoch": 0.65,
395
  "learning_rate": 1.6544367689701824e-06,
396
- "logits/chosen": -2.0102758407592773,
397
- "logits/rejected": -2.006990909576416,
398
- "logps/chosen": -32.49989700317383,
399
- "logps/rejected": -35.99126434326172,
400
- "loss": 0.472,
401
- "rewards/accuracies": 0.5625,
402
- "rewards/chosen": 0.2551037669181824,
403
- "rewards/margins": 0.11490776389837265,
404
- "rewards/rejected": 0.1401960253715515,
405
  "step": 250
406
  },
407
  {
408
  "epoch": 0.68,
409
  "learning_rate": 1.4445974030621963e-06,
410
- "logits/chosen": -1.877345323562622,
411
- "logits/rejected": -1.8749182224273682,
412
- "logps/chosen": -33.7043571472168,
413
- "logps/rejected": -35.259422302246094,
414
- "loss": 0.4671,
415
- "rewards/accuracies": 0.6499999761581421,
416
- "rewards/chosen": 0.2904246747493744,
417
- "rewards/margins": 0.14330394566059113,
418
- "rewards/rejected": 0.14712072908878326,
419
  "step": 260
420
  },
421
  {
422
  "epoch": 0.7,
423
  "learning_rate": 1.243452991757889e-06,
424
- "logits/chosen": -1.8626378774642944,
425
- "logits/rejected": -1.8601328134536743,
426
- "logps/chosen": -33.90363311767578,
427
- "logps/rejected": -31.588598251342773,
428
- "loss": 0.457,
429
- "rewards/accuracies": 0.612500011920929,
430
- "rewards/chosen": 0.29638105630874634,
431
- "rewards/margins": 0.1867421716451645,
432
- "rewards/rejected": 0.10963888466358185,
433
  "step": 270
434
  },
435
  {
436
  "epoch": 0.73,
437
  "learning_rate": 1.0526606671603523e-06,
438
- "logits/chosen": -1.965528130531311,
439
- "logits/rejected": -1.9551260471343994,
440
- "logps/chosen": -34.74420166015625,
441
- "logps/rejected": -31.622207641601562,
442
- "loss": 0.4442,
443
- "rewards/accuracies": 0.6499999761581421,
444
- "rewards/chosen": 0.3674633204936981,
445
- "rewards/margins": 0.23619195818901062,
446
- "rewards/rejected": 0.1312713772058487,
447
  "step": 280
448
  },
449
  {
450
  "epoch": 0.75,
451
  "learning_rate": 8.737922755071455e-07,
452
- "logits/chosen": -2.0614726543426514,
453
- "logits/rejected": -2.0466229915618896,
454
- "logps/chosen": -30.37356948852539,
455
- "logps/rejected": -32.3351936340332,
456
- "loss": 0.4671,
457
- "rewards/accuracies": 0.6000000238418579,
458
- "rewards/chosen": 0.33033472299575806,
459
- "rewards/margins": 0.14034779369831085,
460
- "rewards/rejected": 0.1899869292974472,
461
  "step": 290
462
  },
463
  {
464
  "epoch": 0.78,
465
  "learning_rate": 7.08321427484816e-07,
466
- "logits/chosen": -1.9327377080917358,
467
- "logits/rejected": -1.9302759170532227,
468
- "logps/chosen": -32.0928955078125,
469
- "logps/rejected": -30.652236938476562,
470
- "loss": 0.41,
471
- "rewards/accuracies": 0.737500011920929,
472
- "rewards/chosen": 0.5005279779434204,
473
- "rewards/margins": 0.4017825126647949,
474
- "rewards/rejected": 0.09874548017978668,
475
  "step": 300
476
  },
477
  {
478
  "epoch": 0.78,
479
- "eval_logits/chosen": -2.2308623790740967,
480
- "eval_logits/rejected": -2.226059913635254,
481
- "eval_logps/chosen": -33.76142501831055,
482
- "eval_logps/rejected": -37.284828186035156,
483
- "eval_loss": 0.4937135577201843,
484
- "eval_rewards/accuracies": 0.5361295938491821,
485
- "eval_rewards/chosen": 0.16387464106082916,
486
- "eval_rewards/margins": 0.02479831501841545,
487
- "eval_rewards/rejected": 0.1390763521194458,
488
- "eval_runtime": 145.6766,
489
  "eval_samples_per_second": 2.355,
490
  "eval_steps_per_second": 0.295,
491
  "step": 300
@@ -493,122 +493,122 @@
493
  {
494
  "epoch": 0.81,
495
  "learning_rate": 5.576113578589035e-07,
496
- "logits/chosen": -1.916757345199585,
497
- "logits/rejected": -1.9136326313018799,
498
- "logps/chosen": -31.0240421295166,
499
- "logps/rejected": -33.553138732910156,
500
- "loss": 0.4465,
501
- "rewards/accuracies": 0.6875,
502
- "rewards/chosen": 0.3475847542285919,
503
- "rewards/margins": 0.23417536914348602,
504
- "rewards/rejected": 0.1134093850851059,
505
  "step": 310
506
  },
507
  {
508
  "epoch": 0.83,
509
  "learning_rate": 4.229036944380913e-07,
510
- "logits/chosen": -1.9669981002807617,
511
- "logits/rejected": -1.9548496007919312,
512
- "logps/chosen": -34.02072525024414,
513
- "logps/rejected": -33.40859603881836,
514
- "loss": 0.4379,
515
- "rewards/accuracies": 0.75,
516
- "rewards/chosen": 0.3452104926109314,
517
- "rewards/margins": 0.26706355810165405,
518
- "rewards/rejected": 0.07814691960811615,
519
  "step": 320
520
  },
521
  {
522
  "epoch": 0.86,
523
  "learning_rate": 3.053082288996112e-07,
524
- "logits/chosen": -2.0023112297058105,
525
- "logits/rejected": -2.0009615421295166,
526
- "logps/chosen": -32.86361312866211,
527
- "logps/rejected": -32.265541076660156,
528
- "loss": 0.4439,
529
- "rewards/accuracies": 0.699999988079071,
530
- "rewards/chosen": 0.37330904603004456,
531
- "rewards/margins": 0.24488253891468048,
532
- "rewards/rejected": 0.12842652201652527,
533
  "step": 330
534
  },
535
  {
536
  "epoch": 0.88,
537
  "learning_rate": 2.0579377374915805e-07,
538
- "logits/chosen": -2.0890090465545654,
539
- "logits/rejected": -2.073373317718506,
540
- "logps/chosen": -33.47917556762695,
541
- "logps/rejected": -32.80946350097656,
542
- "loss": 0.4446,
543
- "rewards/accuracies": 0.675000011920929,
544
- "rewards/chosen": 0.42013058066368103,
545
- "rewards/margins": 0.23498812317848206,
546
- "rewards/rejected": 0.18514248728752136,
547
  "step": 340
548
  },
549
  {
550
  "epoch": 0.91,
551
  "learning_rate": 1.2518018074041684e-07,
552
- "logits/chosen": -1.9615952968597412,
553
- "logits/rejected": -1.9607601165771484,
554
- "logps/chosen": -32.57384490966797,
555
- "logps/rejected": -32.25366973876953,
556
- "loss": 0.4326,
557
- "rewards/accuracies": 0.6875,
558
- "rewards/chosen": 0.4249538779258728,
559
- "rewards/margins": 0.2980697453022003,
560
- "rewards/rejected": 0.12688413262367249,
561
  "step": 350
562
  },
563
  {
564
  "epoch": 0.94,
565
  "learning_rate": 6.41315865106129e-08,
566
- "logits/chosen": -1.9177497625350952,
567
- "logits/rejected": -1.9280321598052979,
568
- "logps/chosen": -31.600200653076172,
569
- "logps/rejected": -35.003135681152344,
570
- "loss": 0.4555,
571
- "rewards/accuracies": 0.6499999761581421,
572
- "rewards/chosen": 0.3626941442489624,
573
- "rewards/margins": 0.19132137298583984,
574
- "rewards/rejected": 0.17137278616428375,
575
  "step": 360
576
  },
577
  {
578
  "epoch": 0.96,
579
  "learning_rate": 2.3150941078050325e-08,
580
- "logits/chosen": -2.0559232234954834,
581
- "logits/rejected": -2.0493996143341064,
582
- "logps/chosen": -33.01537322998047,
583
- "logps/rejected": -28.992477416992188,
584
- "loss": 0.4418,
585
- "rewards/accuracies": 0.7749999761581421,
586
- "rewards/chosen": 0.3819587230682373,
587
- "rewards/margins": 0.2415906935930252,
588
- "rewards/rejected": 0.14036807417869568,
589
  "step": 370
590
  },
591
  {
592
  "epoch": 0.99,
593
  "learning_rate": 2.575864278703266e-09,
594
- "logits/chosen": -1.9163579940795898,
595
- "logits/rejected": -1.9185268878936768,
596
- "logps/chosen": -33.650962829589844,
597
- "logps/rejected": -30.753406524658203,
598
- "loss": 0.4369,
599
- "rewards/accuracies": 0.762499988079071,
600
- "rewards/chosen": 0.3528522849082947,
601
- "rewards/margins": 0.28131863474845886,
602
- "rewards/rejected": 0.07153360545635223,
603
  "step": 380
604
  },
605
  {
606
  "epoch": 1.0,
607
  "step": 385,
608
  "total_flos": 0.0,
609
- "train_loss": 0.46439293204963983,
610
- "train_runtime": 3253.2697,
611
- "train_samples_per_second": 0.946,
612
  "train_steps_per_second": 0.118
613
  }
614
  ],
 
15
  "logits/rejected": -1.7377450466156006,
16
  "logps/chosen": -29.553977966308594,
17
  "logps/rejected": -42.813133239746094,
18
+ "loss": 0.6944,
19
  "rewards/accuracies": 0.0,
20
  "rewards/chosen": 0.0,
21
  "rewards/margins": 0.0,
 
25
  {
26
  "epoch": 0.03,
27
  "learning_rate": 1.282051282051282e-06,
28
+ "logits/chosen": -1.8664441108703613,
29
+ "logits/rejected": -1.8707667589187622,
30
+ "logps/chosen": -36.99871826171875,
31
+ "logps/rejected": -33.66032028198242,
32
+ "loss": 0.6698,
33
+ "rewards/accuracies": 0.5138888955116272,
34
+ "rewards/chosen": 0.0047301375307142735,
35
+ "rewards/margins": 0.022878725081682205,
36
+ "rewards/rejected": -0.018148591741919518,
37
  "step": 10
38
  },
39
  {
40
  "epoch": 0.05,
41
  "learning_rate": 2.564102564102564e-06,
42
+ "logits/chosen": -1.9972795248031616,
43
+ "logits/rejected": -1.999922513961792,
44
+ "logps/chosen": -29.64887046813965,
45
+ "logps/rejected": -29.054330825805664,
46
+ "loss": 0.756,
47
+ "rewards/accuracies": 0.38749998807907104,
48
+ "rewards/chosen": -0.0040070666000247,
49
+ "rewards/margins": -0.009474089369177818,
50
+ "rewards/rejected": 0.0054670218378305435,
51
  "step": 20
52
  },
53
  {
54
  "epoch": 0.08,
55
  "learning_rate": 3.846153846153847e-06,
56
+ "logits/chosen": -1.9204769134521484,
57
+ "logits/rejected": -1.9177768230438232,
58
+ "logps/chosen": -31.404077529907227,
59
+ "logps/rejected": -33.22195053100586,
60
+ "loss": 0.7112,
61
  "rewards/accuracies": 0.550000011920929,
62
+ "rewards/chosen": 0.007212291471660137,
63
+ "rewards/margins": 0.009304212406277657,
64
+ "rewards/rejected": -0.0020919169764965773,
65
  "step": 30
66
  },
67
  {
68
  "epoch": 0.1,
69
  "learning_rate": 4.999896948438434e-06,
70
+ "logits/chosen": -2.0172927379608154,
71
+ "logits/rejected": -2.008545398712158,
72
+ "logps/chosen": -32.57653045654297,
73
+ "logps/rejected": -32.520172119140625,
74
+ "loss": 0.7334,
75
+ "rewards/accuracies": 0.5,
76
+ "rewards/chosen": 0.00016416943981312215,
77
+ "rewards/margins": 0.004665405489504337,
78
+ "rewards/rejected": -0.004501235671341419,
79
  "step": 40
80
  },
81
  {
82
  "epoch": 0.13,
83
  "learning_rate": 4.987541037542187e-06,
84
+ "logits/chosen": -1.8620569705963135,
85
+ "logits/rejected": -1.8512712717056274,
86
+ "logps/chosen": -33.55234146118164,
87
+ "logps/rejected": -35.454689025878906,
88
+ "loss": 0.7458,
89
+ "rewards/accuracies": 0.4749999940395355,
90
+ "rewards/chosen": 0.003082751529291272,
91
+ "rewards/margins": 0.004490534774959087,
92
+ "rewards/rejected": -0.0014077809173613787,
93
  "step": 50
94
  },
95
  {
96
  "epoch": 0.16,
97
  "learning_rate": 4.954691471941119e-06,
98
+ "logits/chosen": -1.940212607383728,
99
+ "logits/rejected": -1.9421584606170654,
100
+ "logps/chosen": -32.56269454956055,
101
+ "logps/rejected": -33.21684265136719,
102
+ "loss": 0.6785,
103
+ "rewards/accuracies": 0.574999988079071,
104
+ "rewards/chosen": 0.022492902353405952,
105
+ "rewards/margins": 0.04977109283208847,
106
+ "rewards/rejected": -0.027278194203972816,
107
  "step": 60
108
  },
109
  {
110
  "epoch": 0.18,
111
  "learning_rate": 4.901618883413549e-06,
112
+ "logits/chosen": -2.0728747844696045,
113
+ "logits/rejected": -2.0778398513793945,
114
+ "logps/chosen": -33.98863983154297,
115
+ "logps/rejected": -36.62779235839844,
116
+ "loss": 0.7641,
117
+ "rewards/accuracies": 0.5,
118
+ "rewards/chosen": -9.316801879322156e-05,
119
+ "rewards/margins": 0.035260457545518875,
120
+ "rewards/rejected": -0.035353630781173706,
121
  "step": 70
122
  },
123
  {
124
  "epoch": 0.21,
125
  "learning_rate": 4.828760511501322e-06,
126
+ "logits/chosen": -1.935132384300232,
127
+ "logits/rejected": -1.9382661581039429,
128
+ "logps/chosen": -34.321903228759766,
129
+ "logps/rejected": -34.638282775878906,
130
+ "loss": 0.6095,
131
+ "rewards/accuracies": 0.612500011920929,
132
+ "rewards/chosen": 0.06354494392871857,
133
+ "rewards/margins": 0.09106875956058502,
134
+ "rewards/rejected": -0.027523810043931007,
135
  "step": 80
136
  },
137
  {
138
  "epoch": 0.23,
139
  "learning_rate": 4.7367166013034295e-06,
140
+ "logits/chosen": -1.9444854259490967,
141
+ "logits/rejected": -1.9490203857421875,
142
+ "logps/chosen": -32.41173553466797,
143
+ "logps/rejected": -32.337852478027344,
144
+ "loss": 0.7499,
145
+ "rewards/accuracies": 0.5874999761581421,
146
+ "rewards/chosen": 0.033411428332328796,
147
+ "rewards/margins": 0.020780622959136963,
148
+ "rewards/rejected": 0.012630810961127281,
149
  "step": 90
150
  },
151
  {
152
  "epoch": 0.26,
153
  "learning_rate": 4.626245458345211e-06,
154
+ "logits/chosen": -2.042165994644165,
155
+ "logits/rejected": -2.0401768684387207,
156
+ "logps/chosen": -32.13690185546875,
157
+ "logps/rejected": -31.2872314453125,
158
+ "loss": 0.5903,
159
+ "rewards/accuracies": 0.6625000238418579,
160
+ "rewards/chosen": 0.06236031651496887,
161
+ "rewards/margins": 0.0749734491109848,
162
+ "rewards/rejected": -0.012613129802048206,
163
  "step": 100
164
  },
165
  {
166
  "epoch": 0.26,
167
+ "eval_logits/chosen": -2.2366247177124023,
168
+ "eval_logits/rejected": -2.231769323348999,
169
+ "eval_logps/chosen": -34.01411819458008,
170
+ "eval_logps/rejected": -37.50873947143555,
171
+ "eval_loss": 0.7449910640716553,
172
+ "eval_rewards/accuracies": 0.5240863561630249,
173
+ "eval_rewards/chosen": 0.01226096786558628,
174
+ "eval_rewards/margins": 0.007533104624599218,
175
+ "eval_rewards/rejected": 0.0047278618440032005,
176
+ "eval_runtime": 145.783,
177
+ "eval_samples_per_second": 2.353,
178
  "eval_steps_per_second": 0.295,
179
  "step": 100
180
  },
181
  {
182
  "epoch": 0.29,
183
  "learning_rate": 4.498257201263691e-06,
184
+ "logits/chosen": -1.9979403018951416,
185
+ "logits/rejected": -1.9955459833145142,
186
+ "logps/chosen": -33.12257766723633,
187
+ "logps/rejected": -33.99717712402344,
188
+ "loss": 0.8547,
189
+ "rewards/accuracies": 0.5874999761581421,
190
+ "rewards/chosen": 0.07274619489908218,
191
+ "rewards/margins": 0.05342693254351616,
192
+ "rewards/rejected": 0.019319258630275726,
193
  "step": 110
194
  },
195
  {
196
  "epoch": 0.31,
197
  "learning_rate": 4.353806263777678e-06,
198
+ "logits/chosen": -2.009084463119507,
199
+ "logits/rejected": -2.0007264614105225,
200
+ "logps/chosen": -32.33968734741211,
201
+ "logps/rejected": -32.13513946533203,
202
+ "loss": 0.7104,
203
+ "rewards/accuracies": 0.5625,
204
+ "rewards/chosen": 0.06312941759824753,
205
+ "rewards/margins": 0.047309644520282745,
206
+ "rewards/rejected": 0.015819773077964783,
207
  "step": 120
208
  },
209
  {
210
  "epoch": 0.34,
211
  "learning_rate": 4.1940827077152755e-06,
212
+ "logits/chosen": -2.0365753173828125,
213
+ "logits/rejected": -2.028564453125,
214
+ "logps/chosen": -30.363454818725586,
215
+ "logps/rejected": -32.06770706176758,
216
+ "loss": 0.71,
217
+ "rewards/accuracies": 0.5,
218
+ "rewards/chosen": 0.06735612452030182,
219
+ "rewards/margins": 0.07617705315351486,
220
+ "rewards/rejected": -0.008820934221148491,
221
  "step": 130
222
  },
223
  {
224
  "epoch": 0.36,
225
  "learning_rate": 4.0204024186666215e-06,
226
+ "logits/chosen": -1.9657623767852783,
227
+ "logits/rejected": -1.976011872291565,
228
+ "logps/chosen": -31.213130950927734,
229
+ "logps/rejected": -32.592655181884766,
230
+ "loss": 0.6309,
231
+ "rewards/accuracies": 0.675000011920929,
232
+ "rewards/chosen": 0.12391922622919083,
233
+ "rewards/margins": 0.1585826873779297,
234
+ "rewards/rejected": -0.03466347977519035,
235
  "step": 140
236
  },
237
  {
238
  "epoch": 0.39,
239
  "learning_rate": 3.834196265035119e-06,
240
+ "logits/chosen": -1.8778575658798218,
241
+ "logits/rejected": -1.879020094871521,
242
+ "logps/chosen": -33.998287200927734,
243
+ "logps/rejected": -34.83546829223633,
244
+ "loss": 0.566,
245
+ "rewards/accuracies": 0.6499999761581421,
246
+ "rewards/chosen": 0.11664185672998428,
247
+ "rewards/margins": 0.1635705530643463,
248
+ "rewards/rejected": -0.04692870378494263,
249
  "step": 150
250
  },
251
  {
252
  "epoch": 0.42,
253
  "learning_rate": 3.636998309800573e-06,
254
+ "logits/chosen": -1.9302040338516235,
255
+ "logits/rejected": -1.9267680644989014,
256
+ "logps/chosen": -36.0306510925293,
257
+ "logps/rejected": -32.73199462890625,
258
+ "loss": 0.5795,
259
+ "rewards/accuracies": 0.7250000238418579,
260
+ "rewards/chosen": 0.08461670577526093,
261
+ "rewards/margins": 0.09016285836696625,
262
+ "rewards/rejected": -0.005546141881495714,
263
  "step": 160
264
  },
265
  {
266
  "epoch": 0.44,
267
  "learning_rate": 3.4304331721118078e-06,
268
+ "logits/chosen": -2.031188726425171,
269
+ "logits/rejected": -2.0238261222839355,
270
+ "logps/chosen": -33.55583953857422,
271
+ "logps/rejected": -31.381237030029297,
272
+ "loss": 0.5041,
273
+ "rewards/accuracies": 0.675000011920929,
274
+ "rewards/chosen": 0.14521491527557373,
275
+ "rewards/margins": 0.16284644603729248,
276
+ "rewards/rejected": -0.017631549388170242,
277
  "step": 170
278
  },
279
  {
280
  "epoch": 0.47,
281
  "learning_rate": 3.2162026428305436e-06,
282
+ "logits/chosen": -2.037135362625122,
283
+ "logits/rejected": -2.0424118041992188,
284
+ "logps/chosen": -32.33055114746094,
285
+ "logps/rejected": -32.500144958496094,
286
+ "loss": 0.5257,
287
+ "rewards/accuracies": 0.6875,
288
+ "rewards/chosen": 0.12854143977165222,
289
+ "rewards/margins": 0.13754449784755707,
290
+ "rewards/rejected": -0.009003056213259697,
291
  "step": 180
292
  },
293
  {
294
  "epoch": 0.49,
295
  "learning_rate": 2.996071664294641e-06,
296
+ "logits/chosen": -2.0384888648986816,
297
+ "logits/rejected": -2.0357367992401123,
298
+ "logps/chosen": -31.326950073242188,
299
+ "logps/rejected": -31.338672637939453,
300
+ "loss": 0.6504,
301
+ "rewards/accuracies": 0.625,
302
+ "rewards/chosen": 0.09720095247030258,
303
+ "rewards/margins": 0.10947537422180176,
304
+ "rewards/rejected": -0.012274429202079773,
305
  "step": 190
306
  },
307
  {
308
  "epoch": 0.52,
309
  "learning_rate": 2.7718537898066833e-06,
310
+ "logits/chosen": -1.908755898475647,
311
+ "logits/rejected": -1.9134235382080078,
312
+ "logps/chosen": -31.36275863647461,
313
+ "logps/rejected": -32.82594680786133,
314
+ "loss": 0.59,
315
+ "rewards/accuracies": 0.6875,
316
+ "rewards/chosen": 0.14611336588859558,
317
+ "rewards/margins": 0.16712167859077454,
318
+ "rewards/rejected": -0.02100830152630806,
319
  "step": 200
320
  },
321
  {
322
  "epoch": 0.52,
323
+ "eval_logits/chosen": -2.233200788497925,
324
+ "eval_logits/rejected": -2.228351354598999,
325
+ "eval_logps/chosen": -34.069149017333984,
326
+ "eval_logps/rejected": -37.57521057128906,
327
+ "eval_loss": 0.750046968460083,
328
+ "eval_rewards/accuracies": 0.5365448594093323,
329
+ "eval_rewards/chosen": -0.020758595317602158,
330
+ "eval_rewards/margins": 0.014396170154213905,
331
+ "eval_rewards/rejected": -0.035154763609170914,
332
+ "eval_runtime": 145.6956,
333
+ "eval_samples_per_second": 2.354,
334
  "eval_steps_per_second": 0.295,
335
  "step": 200
336
  },
337
  {
338
  "epoch": 0.55,
339
  "learning_rate": 2.5453962426402006e-06,
340
+ "logits/chosen": -2.0207877159118652,
341
+ "logits/rejected": -2.0314865112304688,
342
+ "logps/chosen": -31.7946834564209,
343
+ "logps/rejected": -33.94416046142578,
344
+ "loss": 0.544,
345
+ "rewards/accuracies": 0.6625000238418579,
346
+ "rewards/chosen": 0.10597337782382965,
347
+ "rewards/margins": 0.1548517644405365,
348
+ "rewards/rejected": -0.04887838289141655,
349
  "step": 210
350
  },
351
  {
352
  "epoch": 0.57,
353
  "learning_rate": 2.3185646976551794e-06,
354
+ "logits/chosen": -1.9139292240142822,
355
+ "logits/rejected": -1.9287306070327759,
356
+ "logps/chosen": -29.91892433166504,
357
+ "logps/rejected": -31.5737247467041,
358
+ "loss": 0.5694,
359
+ "rewards/accuracies": 0.675000011920929,
360
+ "rewards/chosen": 0.11556863784790039,
361
+ "rewards/margins": 0.12489677965641022,
362
+ "rewards/rejected": -0.00932813435792923,
363
  "step": 220
364
  },
365
  {
366
  "epoch": 0.6,
367
  "learning_rate": 2.0932279108998323e-06,
368
+ "logits/chosen": -1.9701712131500244,
369
+ "logits/rejected": -1.974172592163086,
370
+ "logps/chosen": -33.17791748046875,
371
+ "logps/rejected": -31.603836059570312,
372
+ "loss": 0.4933,
373
  "rewards/accuracies": 0.7250000238418579,
374
+ "rewards/chosen": 0.1437317281961441,
375
+ "rewards/margins": 0.1764504611492157,
376
+ "rewards/rejected": -0.03271873667836189,
377
  "step": 230
378
  },
379
  {
380
  "epoch": 0.62,
381
  "learning_rate": 1.8712423238279358e-06,
382
+ "logits/chosen": -1.9691722393035889,
383
+ "logits/rejected": -1.9472358226776123,
384
+ "logps/chosen": -33.92795181274414,
385
+ "logps/rejected": -35.06544876098633,
386
+ "loss": 0.4966,
387
+ "rewards/accuracies": 0.762499988079071,
388
+ "rewards/chosen": 0.13547900319099426,
389
+ "rewards/margins": 0.20427203178405762,
390
+ "rewards/rejected": -0.06879300624132156,
391
  "step": 240
392
  },
393
  {
394
  "epoch": 0.65,
395
  "learning_rate": 1.6544367689701824e-06,
396
+ "logits/chosen": -2.010617971420288,
397
+ "logits/rejected": -2.0072851181030273,
398
+ "logps/chosen": -32.785011291503906,
399
+ "logps/rejected": -36.283348083496094,
400
+ "loss": 0.5823,
401
+ "rewards/accuracies": 0.6875,
402
+ "rewards/chosen": 0.08403309434652328,
403
+ "rewards/margins": 0.11908762156963348,
404
+ "rewards/rejected": -0.035054538398981094,
405
  "step": 250
406
  },
407
  {
408
  "epoch": 0.68,
409
  "learning_rate": 1.4445974030621963e-06,
410
+ "logits/chosen": -1.8787453174591064,
411
+ "logits/rejected": -1.8763244152069092,
412
+ "logps/chosen": -34.000633239746094,
413
+ "logps/rejected": -35.53338623046875,
414
+ "loss": 0.5277,
415
+ "rewards/accuracies": 0.75,
416
+ "rewards/chosen": 0.11265683174133301,
417
+ "rewards/margins": 0.1299169510602951,
418
+ "rewards/rejected": -0.017260130494832993,
419
  "step": 260
420
  },
421
  {
422
  "epoch": 0.7,
423
  "learning_rate": 1.243452991757889e-06,
424
+ "logits/chosen": -1.864057183265686,
425
+ "logits/rejected": -1.861581802368164,
426
+ "logps/chosen": -34.23812484741211,
427
+ "logps/rejected": -31.777530670166016,
428
+ "loss": 0.6307,
429
+ "rewards/accuracies": 0.6625000238418579,
430
+ "rewards/chosen": 0.09568403661251068,
431
+ "rewards/margins": 0.09940369427204132,
432
+ "rewards/rejected": -0.003719669533893466,
433
  "step": 270
434
  },
435
  {
436
  "epoch": 0.73,
437
  "learning_rate": 1.0526606671603523e-06,
438
+ "logits/chosen": -1.9680726528167725,
439
+ "logits/rejected": -1.957493782043457,
440
+ "logps/chosen": -35.05353546142578,
441
+ "logps/rejected": -31.8691463470459,
442
+ "loss": 0.4252,
443
+ "rewards/accuracies": 0.75,
444
+ "rewards/chosen": 0.18186399340629578,
445
+ "rewards/margins": 0.19875726103782654,
446
+ "rewards/rejected": -0.01689324900507927,
447
  "step": 280
448
  },
449
  {
450
  "epoch": 0.75,
451
  "learning_rate": 8.737922755071455e-07,
452
+ "logits/chosen": -2.0636587142944336,
453
+ "logits/rejected": -2.048659563064575,
454
+ "logps/chosen": -30.76055908203125,
455
+ "logps/rejected": -32.65589141845703,
456
+ "loss": 0.6667,
457
+ "rewards/accuracies": 0.625,
458
+ "rewards/chosen": 0.09814020991325378,
459
+ "rewards/margins": 0.10057313740253448,
460
+ "rewards/rejected": -0.0024329207371920347,
461
  "step": 290
462
  },
463
  {
464
  "epoch": 0.78,
465
  "learning_rate": 7.08321427484816e-07,
466
+ "logits/chosen": -1.9355440139770508,
467
+ "logits/rejected": -1.9329853057861328,
468
+ "logps/chosen": -32.57345199584961,
469
+ "logps/rejected": -30.86557388305664,
470
+ "loss": 0.3871,
471
+ "rewards/accuracies": 0.7875000238418579,
472
+ "rewards/chosen": 0.21219508349895477,
473
+ "rewards/margins": 0.24145305156707764,
474
+ "rewards/rejected": -0.029257934540510178,
475
  "step": 300
476
  },
477
  {
478
  "epoch": 0.78,
479
+ "eval_logits/chosen": -2.233344554901123,
480
+ "eval_logits/rejected": -2.2284841537475586,
481
+ "eval_logps/chosen": -34.06221008300781,
482
+ "eval_logps/rejected": -37.56574249267578,
483
+ "eval_loss": 0.7655012607574463,
484
+ "eval_rewards/accuracies": 0.5070598125457764,
485
+ "eval_rewards/chosen": -0.016592998057603836,
486
+ "eval_rewards/margins": 0.012880506925284863,
487
+ "eval_rewards/rejected": -0.029473504051566124,
488
+ "eval_runtime": 145.6769,
489
  "eval_samples_per_second": 2.355,
490
  "eval_steps_per_second": 0.295,
491
  "step": 300
 
493
  {
494
  "epoch": 0.81,
495
  "learning_rate": 5.576113578589035e-07,
496
+ "logits/chosen": -1.9186370372772217,
497
+ "logits/rejected": -1.9153635501861572,
498
+ "logps/chosen": -31.363632202148438,
499
+ "logps/rejected": -33.790061950683594,
500
+ "loss": 0.5232,
501
+ "rewards/accuracies": 0.7124999761581421,
502
+ "rewards/chosen": 0.1438317447900772,
503
+ "rewards/margins": 0.17257718741893768,
504
+ "rewards/rejected": -0.02874544821679592,
505
  "step": 310
506
  },
507
  {
508
  "epoch": 0.83,
509
  "learning_rate": 4.229036944380913e-07,
510
+ "logits/chosen": -1.9695978164672852,
511
+ "logits/rejected": -1.9573185443878174,
512
+ "logps/chosen": -34.42280960083008,
513
+ "logps/rejected": -33.63968276977539,
514
+ "loss": 0.4864,
515
+ "rewards/accuracies": 0.6875,
516
+ "rewards/chosen": 0.103958860039711,
517
+ "rewards/margins": 0.1644643247127533,
518
+ "rewards/rejected": -0.06050548702478409,
519
  "step": 320
520
  },
521
  {
522
  "epoch": 0.86,
523
  "learning_rate": 3.053082288996112e-07,
524
+ "logits/chosen": -2.0046210289001465,
525
+ "logits/rejected": -2.003176689147949,
526
+ "logps/chosen": -33.34724426269531,
527
+ "logps/rejected": -32.53419876098633,
528
+ "loss": 0.5611,
529
+ "rewards/accuracies": 0.675000011920929,
530
+ "rewards/chosen": 0.08312927186489105,
531
+ "rewards/margins": 0.11589743942022324,
532
+ "rewards/rejected": -0.032768163830041885,
533
  "step": 330
534
  },
535
  {
536
  "epoch": 0.88,
537
  "learning_rate": 2.0579377374915805e-07,
538
+ "logits/chosen": -2.091379404067993,
539
+ "logits/rejected": -2.0756208896636963,
540
+ "logps/chosen": -33.90237808227539,
541
+ "logps/rejected": -33.127288818359375,
542
+ "loss": 0.4859,
543
+ "rewards/accuracies": 0.737500011920929,
544
+ "rewards/chosen": 0.16620711982250214,
545
+ "rewards/margins": 0.17176242172718048,
546
+ "rewards/rejected": -0.005555284209549427,
547
  "step": 340
548
  },
549
  {
550
  "epoch": 0.91,
551
  "learning_rate": 1.2518018074041684e-07,
552
+ "logits/chosen": -1.9639079570770264,
553
+ "logits/rejected": -1.9629888534545898,
554
+ "logps/chosen": -32.960304260253906,
555
+ "logps/rejected": -32.5081787109375,
556
+ "loss": 0.5048,
557
+ "rewards/accuracies": 0.762499988079071,
558
+ "rewards/chosen": 0.1930757313966751,
559
+ "rewards/margins": 0.21889439225196838,
560
+ "rewards/rejected": -0.02581869065761566,
561
  "step": 350
562
  },
563
  {
564
  "epoch": 0.94,
565
  "learning_rate": 6.41315865106129e-08,
566
+ "logits/chosen": -1.9195839166641235,
567
+ "logits/rejected": -1.929896593093872,
568
+ "logps/chosen": -31.997264862060547,
569
+ "logps/rejected": -35.334999084472656,
570
+ "loss": 0.5364,
571
+ "rewards/accuracies": 0.7250000238418579,
572
+ "rewards/chosen": 0.12445466220378876,
573
+ "rewards/margins": 0.15219925343990326,
574
+ "rewards/rejected": -0.027744591236114502,
575
  "step": 360
576
  },
577
  {
578
  "epoch": 0.96,
579
  "learning_rate": 2.3150941078050325e-08,
580
+ "logits/chosen": -2.058486223220825,
581
+ "logits/rejected": -2.051971912384033,
582
+ "logps/chosen": -33.44487380981445,
583
+ "logps/rejected": -29.261219024658203,
584
+ "loss": 0.4692,
585
+ "rewards/accuracies": 0.75,
586
+ "rewards/chosen": 0.12425780296325684,
587
+ "rewards/margins": 0.1451319307088852,
588
+ "rewards/rejected": -0.020874138921499252,
589
  "step": 370
590
  },
591
  {
592
  "epoch": 0.99,
593
  "learning_rate": 2.575864278703266e-09,
594
+ "logits/chosen": -1.9182488918304443,
595
+ "logits/rejected": -1.9204342365264893,
596
+ "logps/chosen": -33.968048095703125,
597
+ "logps/rejected": -30.890918731689453,
598
+ "loss": 0.4679,
599
+ "rewards/accuracies": 0.7250000238418579,
600
+ "rewards/chosen": 0.16260387003421783,
601
+ "rewards/margins": 0.17357683181762695,
602
+ "rewards/rejected": -0.010972961783409119,
603
  "step": 380
604
  },
605
  {
606
  "epoch": 1.0,
607
  "step": 385,
608
  "total_flos": 0.0,
609
+ "train_loss": 0.5965218513042896,
610
+ "train_runtime": 3252.2803,
611
+ "train_samples_per_second": 0.947,
612
  "train_steps_per_second": 0.118
613
  }
614
  ],