GaetanMichelet commited on
Commit
665cd21
1 Parent(s): f90adb4

Model save

Browse files
README.md ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ base_model: mistralai/Mistral-7B-Instruct-v0.3
3
+ library_name: peft
4
+ license: apache-2.0
5
+ tags:
6
+ - trl
7
+ - sft
8
+ - generated_from_trainer
9
+ model-index:
10
+ - name: Mistral-7B_task-1_180-samples_config-2
11
+ results: []
12
+ ---
13
+
14
+ <!-- This model card has been generated automatically according to the information the Trainer had access to. You
15
+ should probably proofread and complete it, then remove this comment. -->
16
+
17
+ # Mistral-7B_task-1_180-samples_config-2
18
+
19
+ This model is a fine-tuned version of [mistralai/Mistral-7B-Instruct-v0.3](https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3) on the None dataset.
20
+ It achieves the following results on the evaluation set:
21
+ - Loss: 2.1511
22
+
23
+ ## Model description
24
+
25
+ More information needed
26
+
27
+ ## Intended uses & limitations
28
+
29
+ More information needed
30
+
31
+ ## Training and evaluation data
32
+
33
+ More information needed
34
+
35
+ ## Training procedure
36
+
37
+ ### Training hyperparameters
38
+
39
+ The following hyperparameters were used during training:
40
+ - learning_rate: 0.0001
41
+ - train_batch_size: 1
42
+ - eval_batch_size: 1
43
+ - seed: 42
44
+ - distributed_type: multi-GPU
45
+ - gradient_accumulation_steps: 16
46
+ - total_train_batch_size: 16
47
+ - optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
48
+ - lr_scheduler_type: cosine
49
+ - lr_scheduler_warmup_ratio: 0.1
50
+ - num_epochs: 50
51
+
52
+ ### Training results
53
+
54
+ | Training Loss | Epoch | Step | Validation Loss |
55
+ |:-------------:|:-------:|:----:|:---------------:|
56
+ | 2.053 | 0.9412 | 8 | 1.8890 |
57
+ | 1.3317 | 2.0 | 17 | 1.3413 |
58
+ | 1.1994 | 2.9412 | 25 | 1.2018 |
59
+ | 0.9607 | 4.0 | 34 | 1.1521 |
60
+ | 0.7457 | 4.9412 | 42 | 1.1807 |
61
+ | 0.5441 | 6.0 | 51 | 1.3877 |
62
+ | 0.262 | 6.9412 | 59 | 1.6165 |
63
+ | 0.1407 | 8.0 | 68 | 1.9110 |
64
+ | 0.0589 | 8.9412 | 76 | 1.9831 |
65
+ | 0.053 | 10.0 | 85 | 2.0235 |
66
+ | 0.0268 | 10.9412 | 93 | 2.1511 |
67
+
68
+
69
+ ### Framework versions
70
+
71
+ - PEFT 0.12.0
72
+ - Transformers 4.44.0
73
+ - Pytorch 2.1.2+cu121
74
+ - Datasets 2.20.0
75
+ - Tokenizers 0.19.1
adapter_model.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:967fc1caba8b5325c8d995e7846c93170ab7acdab1f51aa66372d8e1b7a8b396
3
  size 167832240
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d9d33048b308f832b9b9c0885fa8e05d4415bff59ecb7645428ab65924ccc836
3
  size 167832240
all_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.941176470588236,
3
+ "total_flos": 2.3461897328132096e+16,
4
+ "train_loss": 0.709120698954149,
5
+ "train_runtime": 1774.2454,
6
+ "train_samples": 136,
7
+ "train_samples_per_second": 3.833,
8
+ "train_steps_per_second": 0.225
9
+ }
train_results.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "epoch": 10.941176470588236,
3
+ "total_flos": 2.3461897328132096e+16,
4
+ "train_loss": 0.709120698954149,
5
+ "train_runtime": 1774.2454,
6
+ "train_samples": 136,
7
+ "train_samples_per_second": 3.833,
8
+ "train_steps_per_second": 0.225
9
+ }
trainer_state.json ADDED
@@ -0,0 +1,468 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 1.1520808935165405,
3
+ "best_model_checkpoint": "data/Mistral-7B_task-1_180-samples_config-2/checkpoint-34",
4
+ "epoch": 10.941176470588236,
5
+ "eval_steps": 500,
6
+ "global_step": 93,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.11764705882352941,
13
+ "grad_norm": 4.991026401519775,
14
+ "learning_rate": 2.5e-06,
15
+ "loss": 2.1553,
16
+ "step": 1
17
+ },
18
+ {
19
+ "epoch": 0.23529411764705882,
20
+ "grad_norm": 5.117615699768066,
21
+ "learning_rate": 5e-06,
22
+ "loss": 2.1988,
23
+ "step": 2
24
+ },
25
+ {
26
+ "epoch": 0.47058823529411764,
27
+ "grad_norm": 5.423519134521484,
28
+ "learning_rate": 1e-05,
29
+ "loss": 2.2826,
30
+ "step": 4
31
+ },
32
+ {
33
+ "epoch": 0.7058823529411765,
34
+ "grad_norm": 4.905459403991699,
35
+ "learning_rate": 1.5e-05,
36
+ "loss": 2.1742,
37
+ "step": 6
38
+ },
39
+ {
40
+ "epoch": 0.9411764705882353,
41
+ "grad_norm": 4.527350425720215,
42
+ "learning_rate": 2e-05,
43
+ "loss": 2.053,
44
+ "step": 8
45
+ },
46
+ {
47
+ "epoch": 0.9411764705882353,
48
+ "eval_loss": 1.8889563083648682,
49
+ "eval_runtime": 14.5533,
50
+ "eval_samples_per_second": 2.474,
51
+ "eval_steps_per_second": 2.474,
52
+ "step": 8
53
+ },
54
+ {
55
+ "epoch": 1.1764705882352942,
56
+ "grad_norm": 3.475832223892212,
57
+ "learning_rate": 2.5e-05,
58
+ "loss": 1.8598,
59
+ "step": 10
60
+ },
61
+ {
62
+ "epoch": 1.4117647058823528,
63
+ "grad_norm": 2.5801305770874023,
64
+ "learning_rate": 3e-05,
65
+ "loss": 1.6493,
66
+ "step": 12
67
+ },
68
+ {
69
+ "epoch": 1.6470588235294117,
70
+ "grad_norm": 1.8207895755767822,
71
+ "learning_rate": 3.5e-05,
72
+ "loss": 1.4821,
73
+ "step": 14
74
+ },
75
+ {
76
+ "epoch": 1.8823529411764706,
77
+ "grad_norm": 1.4556984901428223,
78
+ "learning_rate": 4e-05,
79
+ "loss": 1.3317,
80
+ "step": 16
81
+ },
82
+ {
83
+ "epoch": 2.0,
84
+ "eval_loss": 1.3413469791412354,
85
+ "eval_runtime": 14.5537,
86
+ "eval_samples_per_second": 2.474,
87
+ "eval_steps_per_second": 2.474,
88
+ "step": 17
89
+ },
90
+ {
91
+ "epoch": 2.1176470588235294,
92
+ "grad_norm": 1.04146409034729,
93
+ "learning_rate": 4.5e-05,
94
+ "loss": 1.2367,
95
+ "step": 18
96
+ },
97
+ {
98
+ "epoch": 2.3529411764705883,
99
+ "grad_norm": 1.2536530494689941,
100
+ "learning_rate": 5e-05,
101
+ "loss": 1.2383,
102
+ "step": 20
103
+ },
104
+ {
105
+ "epoch": 2.588235294117647,
106
+ "grad_norm": 1.1814831495285034,
107
+ "learning_rate": 5.500000000000001e-05,
108
+ "loss": 1.2719,
109
+ "step": 22
110
+ },
111
+ {
112
+ "epoch": 2.8235294117647056,
113
+ "grad_norm": 1.0946425199508667,
114
+ "learning_rate": 6e-05,
115
+ "loss": 1.1994,
116
+ "step": 24
117
+ },
118
+ {
119
+ "epoch": 2.9411764705882355,
120
+ "eval_loss": 1.2018373012542725,
121
+ "eval_runtime": 14.5629,
122
+ "eval_samples_per_second": 2.472,
123
+ "eval_steps_per_second": 2.472,
124
+ "step": 25
125
+ },
126
+ {
127
+ "epoch": 3.0588235294117645,
128
+ "grad_norm": 1.0643476247787476,
129
+ "learning_rate": 6.500000000000001e-05,
130
+ "loss": 1.1306,
131
+ "step": 26
132
+ },
133
+ {
134
+ "epoch": 3.2941176470588234,
135
+ "grad_norm": 1.1662391424179077,
136
+ "learning_rate": 7e-05,
137
+ "loss": 1.0472,
138
+ "step": 28
139
+ },
140
+ {
141
+ "epoch": 3.5294117647058822,
142
+ "grad_norm": 1.2767999172210693,
143
+ "learning_rate": 7.500000000000001e-05,
144
+ "loss": 1.0429,
145
+ "step": 30
146
+ },
147
+ {
148
+ "epoch": 3.764705882352941,
149
+ "grad_norm": 1.1417080163955688,
150
+ "learning_rate": 8e-05,
151
+ "loss": 0.9668,
152
+ "step": 32
153
+ },
154
+ {
155
+ "epoch": 4.0,
156
+ "grad_norm": 1.3573873043060303,
157
+ "learning_rate": 8.5e-05,
158
+ "loss": 0.9607,
159
+ "step": 34
160
+ },
161
+ {
162
+ "epoch": 4.0,
163
+ "eval_loss": 1.1520808935165405,
164
+ "eval_runtime": 14.5602,
165
+ "eval_samples_per_second": 2.472,
166
+ "eval_steps_per_second": 2.472,
167
+ "step": 34
168
+ },
169
+ {
170
+ "epoch": 4.235294117647059,
171
+ "grad_norm": 1.1198458671569824,
172
+ "learning_rate": 9e-05,
173
+ "loss": 0.8712,
174
+ "step": 36
175
+ },
176
+ {
177
+ "epoch": 4.470588235294118,
178
+ "grad_norm": 1.2332489490509033,
179
+ "learning_rate": 9.5e-05,
180
+ "loss": 0.8081,
181
+ "step": 38
182
+ },
183
+ {
184
+ "epoch": 4.705882352941177,
185
+ "grad_norm": 1.2247358560562134,
186
+ "learning_rate": 0.0001,
187
+ "loss": 0.7503,
188
+ "step": 40
189
+ },
190
+ {
191
+ "epoch": 4.9411764705882355,
192
+ "grad_norm": 1.4213950634002686,
193
+ "learning_rate": 9.999238475781957e-05,
194
+ "loss": 0.7457,
195
+ "step": 42
196
+ },
197
+ {
198
+ "epoch": 4.9411764705882355,
199
+ "eval_loss": 1.1806774139404297,
200
+ "eval_runtime": 14.5514,
201
+ "eval_samples_per_second": 2.474,
202
+ "eval_steps_per_second": 2.474,
203
+ "step": 42
204
+ },
205
+ {
206
+ "epoch": 5.176470588235294,
207
+ "grad_norm": 1.2573533058166504,
208
+ "learning_rate": 9.99695413509548e-05,
209
+ "loss": 0.5997,
210
+ "step": 44
211
+ },
212
+ {
213
+ "epoch": 5.411764705882353,
214
+ "grad_norm": 1.7378907203674316,
215
+ "learning_rate": 9.99314767377287e-05,
216
+ "loss": 0.5895,
217
+ "step": 46
218
+ },
219
+ {
220
+ "epoch": 5.647058823529412,
221
+ "grad_norm": 1.972646951675415,
222
+ "learning_rate": 9.987820251299122e-05,
223
+ "loss": 0.4286,
224
+ "step": 48
225
+ },
226
+ {
227
+ "epoch": 5.882352941176471,
228
+ "grad_norm": 2.6307058334350586,
229
+ "learning_rate": 9.980973490458728e-05,
230
+ "loss": 0.5441,
231
+ "step": 50
232
+ },
233
+ {
234
+ "epoch": 6.0,
235
+ "eval_loss": 1.3877015113830566,
236
+ "eval_runtime": 14.5569,
237
+ "eval_samples_per_second": 2.473,
238
+ "eval_steps_per_second": 2.473,
239
+ "step": 51
240
+ },
241
+ {
242
+ "epoch": 6.117647058823529,
243
+ "grad_norm": 1.6350064277648926,
244
+ "learning_rate": 9.972609476841367e-05,
245
+ "loss": 0.3851,
246
+ "step": 52
247
+ },
248
+ {
249
+ "epoch": 6.352941176470588,
250
+ "grad_norm": 2.2904036045074463,
251
+ "learning_rate": 9.962730758206611e-05,
252
+ "loss": 0.2898,
253
+ "step": 54
254
+ },
255
+ {
256
+ "epoch": 6.588235294117647,
257
+ "grad_norm": 2.2712900638580322,
258
+ "learning_rate": 9.951340343707852e-05,
259
+ "loss": 0.2675,
260
+ "step": 56
261
+ },
262
+ {
263
+ "epoch": 6.823529411764706,
264
+ "grad_norm": 2.793653726577759,
265
+ "learning_rate": 9.938441702975689e-05,
266
+ "loss": 0.262,
267
+ "step": 58
268
+ },
269
+ {
270
+ "epoch": 6.9411764705882355,
271
+ "eval_loss": 1.616546630859375,
272
+ "eval_runtime": 14.5577,
273
+ "eval_samples_per_second": 2.473,
274
+ "eval_steps_per_second": 2.473,
275
+ "step": 59
276
+ },
277
+ {
278
+ "epoch": 7.0588235294117645,
279
+ "grad_norm": 1.9732766151428223,
280
+ "learning_rate": 9.924038765061042e-05,
281
+ "loss": 0.2309,
282
+ "step": 60
283
+ },
284
+ {
285
+ "epoch": 7.294117647058823,
286
+ "grad_norm": 1.9136290550231934,
287
+ "learning_rate": 9.908135917238321e-05,
288
+ "loss": 0.1441,
289
+ "step": 62
290
+ },
291
+ {
292
+ "epoch": 7.529411764705882,
293
+ "grad_norm": 2.7887516021728516,
294
+ "learning_rate": 9.890738003669029e-05,
295
+ "loss": 0.1155,
296
+ "step": 64
297
+ },
298
+ {
299
+ "epoch": 7.764705882352941,
300
+ "grad_norm": 3.061058282852173,
301
+ "learning_rate": 9.871850323926177e-05,
302
+ "loss": 0.1023,
303
+ "step": 66
304
+ },
305
+ {
306
+ "epoch": 8.0,
307
+ "grad_norm": 3.621833324432373,
308
+ "learning_rate": 9.851478631379982e-05,
309
+ "loss": 0.1407,
310
+ "step": 68
311
+ },
312
+ {
313
+ "epoch": 8.0,
314
+ "eval_loss": 1.9109816551208496,
315
+ "eval_runtime": 14.5636,
316
+ "eval_samples_per_second": 2.472,
317
+ "eval_steps_per_second": 2.472,
318
+ "step": 68
319
+ },
320
+ {
321
+ "epoch": 8.235294117647058,
322
+ "grad_norm": 2.3172683715820312,
323
+ "learning_rate": 9.829629131445342e-05,
324
+ "loss": 0.0603,
325
+ "step": 70
326
+ },
327
+ {
328
+ "epoch": 8.470588235294118,
329
+ "grad_norm": 2.268026351928711,
330
+ "learning_rate": 9.806308479691595e-05,
331
+ "loss": 0.0708,
332
+ "step": 72
333
+ },
334
+ {
335
+ "epoch": 8.705882352941176,
336
+ "grad_norm": 2.028542995452881,
337
+ "learning_rate": 9.781523779815179e-05,
338
+ "loss": 0.0715,
339
+ "step": 74
340
+ },
341
+ {
342
+ "epoch": 8.941176470588236,
343
+ "grad_norm": 1.5232679843902588,
344
+ "learning_rate": 9.755282581475769e-05,
345
+ "loss": 0.0589,
346
+ "step": 76
347
+ },
348
+ {
349
+ "epoch": 8.941176470588236,
350
+ "eval_loss": 1.9830735921859741,
351
+ "eval_runtime": 14.5581,
352
+ "eval_samples_per_second": 2.473,
353
+ "eval_steps_per_second": 2.473,
354
+ "step": 76
355
+ },
356
+ {
357
+ "epoch": 9.176470588235293,
358
+ "grad_norm": 1.2906736135482788,
359
+ "learning_rate": 9.727592877996585e-05,
360
+ "loss": 0.0401,
361
+ "step": 78
362
+ },
363
+ {
364
+ "epoch": 9.411764705882353,
365
+ "grad_norm": 2.3489370346069336,
366
+ "learning_rate": 9.698463103929542e-05,
367
+ "loss": 0.0418,
368
+ "step": 80
369
+ },
370
+ {
371
+ "epoch": 9.647058823529411,
372
+ "grad_norm": 2.143594741821289,
373
+ "learning_rate": 9.667902132486009e-05,
374
+ "loss": 0.0481,
375
+ "step": 82
376
+ },
377
+ {
378
+ "epoch": 9.882352941176471,
379
+ "grad_norm": 1.7526501417160034,
380
+ "learning_rate": 9.635919272833938e-05,
381
+ "loss": 0.053,
382
+ "step": 84
383
+ },
384
+ {
385
+ "epoch": 10.0,
386
+ "eval_loss": 2.023517370223999,
387
+ "eval_runtime": 14.5596,
388
+ "eval_samples_per_second": 2.473,
389
+ "eval_steps_per_second": 2.473,
390
+ "step": 85
391
+ },
392
+ {
393
+ "epoch": 10.117647058823529,
394
+ "grad_norm": 1.686197280883789,
395
+ "learning_rate": 9.602524267262203e-05,
396
+ "loss": 0.0401,
397
+ "step": 86
398
+ },
399
+ {
400
+ "epoch": 10.352941176470589,
401
+ "grad_norm": 1.4375970363616943,
402
+ "learning_rate": 9.567727288213005e-05,
403
+ "loss": 0.026,
404
+ "step": 88
405
+ },
406
+ {
407
+ "epoch": 10.588235294117647,
408
+ "grad_norm": 1.4091485738754272,
409
+ "learning_rate": 9.53153893518325e-05,
410
+ "loss": 0.0297,
411
+ "step": 90
412
+ },
413
+ {
414
+ "epoch": 10.823529411764707,
415
+ "grad_norm": 1.1159406900405884,
416
+ "learning_rate": 9.493970231495835e-05,
417
+ "loss": 0.0268,
418
+ "step": 92
419
+ },
420
+ {
421
+ "epoch": 10.941176470588236,
422
+ "eval_loss": 2.1510894298553467,
423
+ "eval_runtime": 14.5798,
424
+ "eval_samples_per_second": 2.469,
425
+ "eval_steps_per_second": 2.469,
426
+ "step": 93
427
+ },
428
+ {
429
+ "epoch": 10.941176470588236,
430
+ "step": 93,
431
+ "total_flos": 2.3461897328132096e+16,
432
+ "train_loss": 0.709120698954149,
433
+ "train_runtime": 1774.2454,
434
+ "train_samples_per_second": 3.833,
435
+ "train_steps_per_second": 0.225
436
+ }
437
+ ],
438
+ "logging_steps": 2,
439
+ "max_steps": 400,
440
+ "num_input_tokens_seen": 0,
441
+ "num_train_epochs": 50,
442
+ "save_steps": 25,
443
+ "stateful_callbacks": {
444
+ "EarlyStoppingCallback": {
445
+ "args": {
446
+ "early_stopping_patience": 7,
447
+ "early_stopping_threshold": 0.0
448
+ },
449
+ "attributes": {
450
+ "early_stopping_patience_counter": 0
451
+ }
452
+ },
453
+ "TrainerControl": {
454
+ "args": {
455
+ "should_epoch_stop": false,
456
+ "should_evaluate": false,
457
+ "should_log": false,
458
+ "should_save": true,
459
+ "should_training_stop": true
460
+ },
461
+ "attributes": {}
462
+ }
463
+ },
464
+ "total_flos": 2.3461897328132096e+16,
465
+ "train_batch_size": 1,
466
+ "trial_name": null,
467
+ "trial_params": null
468
+ }