ambrim commited on
Commit
8cdfbf9
1 Parent(s): a2111e3

Upload 10 files

Browse files
config.json ADDED
@@ -0,0 +1,30 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "ambrim/finetuned-epitope_attempt7_150M_200k_default_accuracy_more",
3
+ "architectures": [
4
+ "EsmForTokenClassification"
5
+ ],
6
+ "attention_probs_dropout_prob": 0.0,
7
+ "classifier_dropout": null,
8
+ "emb_layer_norm_before": false,
9
+ "esmfold_config": null,
10
+ "hidden_act": "gelu",
11
+ "hidden_dropout_prob": 0.0,
12
+ "hidden_size": 640,
13
+ "initializer_range": 0.02,
14
+ "intermediate_size": 2560,
15
+ "is_folding_model": false,
16
+ "layer_norm_eps": 1e-05,
17
+ "mask_token_id": 32,
18
+ "max_position_embeddings": 1026,
19
+ "model_type": "esm",
20
+ "num_attention_heads": 20,
21
+ "num_hidden_layers": 30,
22
+ "pad_token_id": 1,
23
+ "position_embedding_type": "rotary",
24
+ "token_dropout": true,
25
+ "torch_dtype": "float32",
26
+ "transformers_version": "4.36.0",
27
+ "use_cache": true,
28
+ "vocab_list": null,
29
+ "vocab_size": 33
30
+ }
model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:86d3678ccad548f636d1644abbc8cade2bf13b3247d5899a1b43c1428cdc0838
3
+ size 593608020
optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3f06d96a8400cff776cdc7f4ce49159836b50ccae84a516166348a3acb84e461
3
+ size 1182240577
rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:7c94c375fe5ad2903d244ca6b5cc2a1a6cba4c0c26196f3b9cbd9ddd170bb0b8
3
+ size 14244
scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:384bf607c3c150887ecedeb07e683fb1d492b749a22d9ccdebef895bf42a652b
3
+ size 1064
special_tokens_map.json ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "cls_token": {
3
+ "content": "<cls>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "eos_token": {
10
+ "content": "<eos>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "mask_token": {
17
+ "content": "<mask>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ },
23
+ "pad_token": {
24
+ "content": "<pad>",
25
+ "lstrip": false,
26
+ "normalized": false,
27
+ "rstrip": false,
28
+ "single_word": false
29
+ },
30
+ "unk_token": {
31
+ "content": "<unk>",
32
+ "lstrip": false,
33
+ "normalized": false,
34
+ "rstrip": false,
35
+ "single_word": false
36
+ }
37
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "0": {
4
+ "content": "<cls>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ },
11
+ "1": {
12
+ "content": "<pad>",
13
+ "lstrip": false,
14
+ "normalized": false,
15
+ "rstrip": false,
16
+ "single_word": false,
17
+ "special": true
18
+ },
19
+ "2": {
20
+ "content": "<eos>",
21
+ "lstrip": false,
22
+ "normalized": false,
23
+ "rstrip": false,
24
+ "single_word": false,
25
+ "special": true
26
+ },
27
+ "3": {
28
+ "content": "<unk>",
29
+ "lstrip": false,
30
+ "normalized": false,
31
+ "rstrip": false,
32
+ "single_word": false,
33
+ "special": true
34
+ },
35
+ "32": {
36
+ "content": "<mask>",
37
+ "lstrip": false,
38
+ "normalized": false,
39
+ "rstrip": false,
40
+ "single_word": false,
41
+ "special": true
42
+ }
43
+ },
44
+ "clean_up_tokenization_spaces": true,
45
+ "cls_token": "<cls>",
46
+ "eos_token": "<eos>",
47
+ "mask_token": "<mask>",
48
+ "model_max_length": 1000000000000000019884624838656,
49
+ "pad_token": "<pad>",
50
+ "tokenizer_class": "EsmTokenizer",
51
+ "unk_token": "<unk>"
52
+ }
trainer_state.json ADDED
@@ -0,0 +1,933 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_metric": 0.9479369602362184,
3
+ "best_model_checkpoint": "finetuned-epitope_attempt7_150M_200k_default_accuracy_more-finetuned-epitope_attempt8_150M_200k_default_accuracy_mostest/checkpoint-6862",
4
+ "epoch": 10.0,
5
+ "eval_steps": 500,
6
+ "global_step": 68620,
7
+ "is_hyper_param_search": false,
8
+ "is_local_process_zero": true,
9
+ "is_world_process_zero": true,
10
+ "log_history": [
11
+ {
12
+ "epoch": 0.07,
13
+ "learning_rate": 9.92713494607986e-05,
14
+ "loss": 0.0915,
15
+ "step": 500
16
+ },
17
+ {
18
+ "epoch": 0.15,
19
+ "learning_rate": 9.854269892159721e-05,
20
+ "loss": 0.109,
21
+ "step": 1000
22
+ },
23
+ {
24
+ "epoch": 0.22,
25
+ "learning_rate": 9.781404838239581e-05,
26
+ "loss": 0.1101,
27
+ "step": 1500
28
+ },
29
+ {
30
+ "epoch": 0.29,
31
+ "learning_rate": 9.70853978431944e-05,
32
+ "loss": 0.121,
33
+ "step": 2000
34
+ },
35
+ {
36
+ "epoch": 0.36,
37
+ "learning_rate": 9.635674730399302e-05,
38
+ "loss": 0.1245,
39
+ "step": 2500
40
+ },
41
+ {
42
+ "epoch": 0.44,
43
+ "learning_rate": 9.562809676479161e-05,
44
+ "loss": 0.1231,
45
+ "step": 3000
46
+ },
47
+ {
48
+ "epoch": 0.51,
49
+ "learning_rate": 9.489944622559021e-05,
50
+ "loss": 0.1227,
51
+ "step": 3500
52
+ },
53
+ {
54
+ "epoch": 0.58,
55
+ "learning_rate": 9.417079568638881e-05,
56
+ "loss": 0.1291,
57
+ "step": 4000
58
+ },
59
+ {
60
+ "epoch": 0.66,
61
+ "learning_rate": 9.34421451471874e-05,
62
+ "loss": 0.1283,
63
+ "step": 4500
64
+ },
65
+ {
66
+ "epoch": 0.73,
67
+ "learning_rate": 9.271349460798602e-05,
68
+ "loss": 0.1333,
69
+ "step": 5000
70
+ },
71
+ {
72
+ "epoch": 0.8,
73
+ "learning_rate": 9.198484406878461e-05,
74
+ "loss": 0.1309,
75
+ "step": 5500
76
+ },
77
+ {
78
+ "epoch": 0.87,
79
+ "learning_rate": 9.125619352958321e-05,
80
+ "loss": 0.146,
81
+ "step": 6000
82
+ },
83
+ {
84
+ "epoch": 0.95,
85
+ "learning_rate": 9.052754299038182e-05,
86
+ "loss": 0.1353,
87
+ "step": 6500
88
+ },
89
+ {
90
+ "epoch": 1.0,
91
+ "eval_accuracy": 0.9479369602362184,
92
+ "eval_loss": 0.13793735206127167,
93
+ "eval_runtime": 153.2159,
94
+ "eval_samples_per_second": 29.86,
95
+ "eval_steps_per_second": 4.98,
96
+ "step": 6862
97
+ },
98
+ {
99
+ "epoch": 1.02,
100
+ "learning_rate": 8.979889245118042e-05,
101
+ "loss": 0.1302,
102
+ "step": 7000
103
+ },
104
+ {
105
+ "epoch": 1.09,
106
+ "learning_rate": 8.907024191197903e-05,
107
+ "loss": 0.1158,
108
+ "step": 7500
109
+ },
110
+ {
111
+ "epoch": 1.17,
112
+ "learning_rate": 8.834159137277762e-05,
113
+ "loss": 0.1115,
114
+ "step": 8000
115
+ },
116
+ {
117
+ "epoch": 1.24,
118
+ "learning_rate": 8.761294083357622e-05,
119
+ "loss": 0.1123,
120
+ "step": 8500
121
+ },
122
+ {
123
+ "epoch": 1.31,
124
+ "learning_rate": 8.688429029437483e-05,
125
+ "loss": 0.1219,
126
+ "step": 9000
127
+ },
128
+ {
129
+ "epoch": 1.38,
130
+ "learning_rate": 8.615563975517343e-05,
131
+ "loss": 0.1192,
132
+ "step": 9500
133
+ },
134
+ {
135
+ "epoch": 1.46,
136
+ "learning_rate": 8.542698921597203e-05,
137
+ "loss": 0.124,
138
+ "step": 10000
139
+ },
140
+ {
141
+ "epoch": 1.53,
142
+ "learning_rate": 8.469833867677062e-05,
143
+ "loss": 0.1161,
144
+ "step": 10500
145
+ },
146
+ {
147
+ "epoch": 1.6,
148
+ "learning_rate": 8.396968813756922e-05,
149
+ "loss": 0.1168,
150
+ "step": 11000
151
+ },
152
+ {
153
+ "epoch": 1.68,
154
+ "learning_rate": 8.324103759836782e-05,
155
+ "loss": 0.1235,
156
+ "step": 11500
157
+ },
158
+ {
159
+ "epoch": 1.75,
160
+ "learning_rate": 8.251238705916643e-05,
161
+ "loss": 0.1243,
162
+ "step": 12000
163
+ },
164
+ {
165
+ "epoch": 1.82,
166
+ "learning_rate": 8.178373651996502e-05,
167
+ "loss": 0.1258,
168
+ "step": 12500
169
+ },
170
+ {
171
+ "epoch": 1.89,
172
+ "learning_rate": 8.105508598076364e-05,
173
+ "loss": 0.1292,
174
+ "step": 13000
175
+ },
176
+ {
177
+ "epoch": 1.97,
178
+ "learning_rate": 8.032643544156223e-05,
179
+ "loss": 0.1243,
180
+ "step": 13500
181
+ },
182
+ {
183
+ "epoch": 2.0,
184
+ "eval_accuracy": 0.9455701299304663,
185
+ "eval_loss": 0.15209880471229553,
186
+ "eval_runtime": 153.336,
187
+ "eval_samples_per_second": 29.836,
188
+ "eval_steps_per_second": 4.976,
189
+ "step": 13724
190
+ },
191
+ {
192
+ "epoch": 2.04,
193
+ "learning_rate": 7.959778490236083e-05,
194
+ "loss": 0.1152,
195
+ "step": 14000
196
+ },
197
+ {
198
+ "epoch": 2.11,
199
+ "learning_rate": 7.886913436315944e-05,
200
+ "loss": 0.0952,
201
+ "step": 14500
202
+ },
203
+ {
204
+ "epoch": 2.19,
205
+ "learning_rate": 7.814048382395804e-05,
206
+ "loss": 0.0961,
207
+ "step": 15000
208
+ },
209
+ {
210
+ "epoch": 2.26,
211
+ "learning_rate": 7.741183328475663e-05,
212
+ "loss": 0.1019,
213
+ "step": 15500
214
+ },
215
+ {
216
+ "epoch": 2.33,
217
+ "learning_rate": 7.668318274555523e-05,
218
+ "loss": 0.1097,
219
+ "step": 16000
220
+ },
221
+ {
222
+ "epoch": 2.4,
223
+ "learning_rate": 7.595453220635383e-05,
224
+ "loss": 0.1091,
225
+ "step": 16500
226
+ },
227
+ {
228
+ "epoch": 2.48,
229
+ "learning_rate": 7.522588166715244e-05,
230
+ "loss": 0.1067,
231
+ "step": 17000
232
+ },
233
+ {
234
+ "epoch": 2.55,
235
+ "learning_rate": 7.449723112795104e-05,
236
+ "loss": 0.1112,
237
+ "step": 17500
238
+ },
239
+ {
240
+ "epoch": 2.62,
241
+ "learning_rate": 7.376858058874963e-05,
242
+ "loss": 0.1119,
243
+ "step": 18000
244
+ },
245
+ {
246
+ "epoch": 2.7,
247
+ "learning_rate": 7.303993004954824e-05,
248
+ "loss": 0.1117,
249
+ "step": 18500
250
+ },
251
+ {
252
+ "epoch": 2.77,
253
+ "learning_rate": 7.231127951034684e-05,
254
+ "loss": 0.1112,
255
+ "step": 19000
256
+ },
257
+ {
258
+ "epoch": 2.84,
259
+ "learning_rate": 7.158262897114544e-05,
260
+ "loss": 0.1125,
261
+ "step": 19500
262
+ },
263
+ {
264
+ "epoch": 2.91,
265
+ "learning_rate": 7.085397843194405e-05,
266
+ "loss": 0.1134,
267
+ "step": 20000
268
+ },
269
+ {
270
+ "epoch": 2.99,
271
+ "learning_rate": 7.012532789274264e-05,
272
+ "loss": 0.1146,
273
+ "step": 20500
274
+ },
275
+ {
276
+ "epoch": 3.0,
277
+ "eval_accuracy": 0.9437319003298592,
278
+ "eval_loss": 0.1713561713695526,
279
+ "eval_runtime": 153.3707,
280
+ "eval_samples_per_second": 29.83,
281
+ "eval_steps_per_second": 4.975,
282
+ "step": 20586
283
+ },
284
+ {
285
+ "epoch": 3.06,
286
+ "learning_rate": 6.939667735354124e-05,
287
+ "loss": 0.0969,
288
+ "step": 21000
289
+ },
290
+ {
291
+ "epoch": 3.13,
292
+ "learning_rate": 6.866802681433985e-05,
293
+ "loss": 0.0846,
294
+ "step": 21500
295
+ },
296
+ {
297
+ "epoch": 3.21,
298
+ "learning_rate": 6.793937627513845e-05,
299
+ "loss": 0.0911,
300
+ "step": 22000
301
+ },
302
+ {
303
+ "epoch": 3.28,
304
+ "learning_rate": 6.721072573593705e-05,
305
+ "loss": 0.0915,
306
+ "step": 22500
307
+ },
308
+ {
309
+ "epoch": 3.35,
310
+ "learning_rate": 6.648207519673564e-05,
311
+ "loss": 0.093,
312
+ "step": 23000
313
+ },
314
+ {
315
+ "epoch": 3.42,
316
+ "learning_rate": 6.575342465753424e-05,
317
+ "loss": 0.095,
318
+ "step": 23500
319
+ },
320
+ {
321
+ "epoch": 3.5,
322
+ "learning_rate": 6.502477411833285e-05,
323
+ "loss": 0.1049,
324
+ "step": 24000
325
+ },
326
+ {
327
+ "epoch": 3.57,
328
+ "learning_rate": 6.429612357913145e-05,
329
+ "loss": 0.0957,
330
+ "step": 24500
331
+ },
332
+ {
333
+ "epoch": 3.64,
334
+ "learning_rate": 6.356747303993004e-05,
335
+ "loss": 0.0982,
336
+ "step": 25000
337
+ },
338
+ {
339
+ "epoch": 3.72,
340
+ "learning_rate": 6.283882250072866e-05,
341
+ "loss": 0.0996,
342
+ "step": 25500
343
+ },
344
+ {
345
+ "epoch": 3.79,
346
+ "learning_rate": 6.211017196152725e-05,
347
+ "loss": 0.1034,
348
+ "step": 26000
349
+ },
350
+ {
351
+ "epoch": 3.86,
352
+ "learning_rate": 6.138152142232586e-05,
353
+ "loss": 0.0981,
354
+ "step": 26500
355
+ },
356
+ {
357
+ "epoch": 3.93,
358
+ "learning_rate": 6.065287088312446e-05,
359
+ "loss": 0.1001,
360
+ "step": 27000
361
+ },
362
+ {
363
+ "epoch": 4.0,
364
+ "eval_accuracy": 0.9392775102080849,
365
+ "eval_loss": 0.19221451878547668,
366
+ "eval_runtime": 153.3316,
367
+ "eval_samples_per_second": 29.837,
368
+ "eval_steps_per_second": 4.976,
369
+ "step": 27448
370
+ },
371
+ {
372
+ "epoch": 4.01,
373
+ "learning_rate": 5.992422034392306e-05,
374
+ "loss": 0.0978,
375
+ "step": 27500
376
+ },
377
+ {
378
+ "epoch": 4.08,
379
+ "learning_rate": 5.919556980472166e-05,
380
+ "loss": 0.0833,
381
+ "step": 28000
382
+ },
383
+ {
384
+ "epoch": 4.15,
385
+ "learning_rate": 5.846691926552026e-05,
386
+ "loss": 0.0839,
387
+ "step": 28500
388
+ },
389
+ {
390
+ "epoch": 4.23,
391
+ "learning_rate": 5.7738268726318855e-05,
392
+ "loss": 0.0789,
393
+ "step": 29000
394
+ },
395
+ {
396
+ "epoch": 4.3,
397
+ "learning_rate": 5.7009618187117465e-05,
398
+ "loss": 0.083,
399
+ "step": 29500
400
+ },
401
+ {
402
+ "epoch": 4.37,
403
+ "learning_rate": 5.628096764791606e-05,
404
+ "loss": 0.0897,
405
+ "step": 30000
406
+ },
407
+ {
408
+ "epoch": 4.44,
409
+ "learning_rate": 5.555231710871466e-05,
410
+ "loss": 0.0836,
411
+ "step": 30500
412
+ },
413
+ {
414
+ "epoch": 4.52,
415
+ "learning_rate": 5.482366656951326e-05,
416
+ "loss": 0.0863,
417
+ "step": 31000
418
+ },
419
+ {
420
+ "epoch": 4.59,
421
+ "learning_rate": 5.409501603031186e-05,
422
+ "loss": 0.0878,
423
+ "step": 31500
424
+ },
425
+ {
426
+ "epoch": 4.66,
427
+ "learning_rate": 5.336636549111047e-05,
428
+ "loss": 0.0861,
429
+ "step": 32000
430
+ },
431
+ {
432
+ "epoch": 4.74,
433
+ "learning_rate": 5.263771495190907e-05,
434
+ "loss": 0.0955,
435
+ "step": 32500
436
+ },
437
+ {
438
+ "epoch": 4.81,
439
+ "learning_rate": 5.1909064412707665e-05,
440
+ "loss": 0.0874,
441
+ "step": 33000
442
+ },
443
+ {
444
+ "epoch": 4.88,
445
+ "learning_rate": 5.1180413873506275e-05,
446
+ "loss": 0.0926,
447
+ "step": 33500
448
+ },
449
+ {
450
+ "epoch": 4.95,
451
+ "learning_rate": 5.045176333430487e-05,
452
+ "loss": 0.0861,
453
+ "step": 34000
454
+ },
455
+ {
456
+ "epoch": 5.0,
457
+ "eval_accuracy": 0.9366432822800035,
458
+ "eval_loss": 0.23794159293174744,
459
+ "eval_runtime": 153.3625,
460
+ "eval_samples_per_second": 29.831,
461
+ "eval_steps_per_second": 4.975,
462
+ "step": 34310
463
+ },
464
+ {
465
+ "epoch": 5.03,
466
+ "learning_rate": 4.972311279510347e-05,
467
+ "loss": 0.0822,
468
+ "step": 34500
469
+ },
470
+ {
471
+ "epoch": 5.1,
472
+ "learning_rate": 4.899446225590207e-05,
473
+ "loss": 0.0734,
474
+ "step": 35000
475
+ },
476
+ {
477
+ "epoch": 5.17,
478
+ "learning_rate": 4.826581171670067e-05,
479
+ "loss": 0.0713,
480
+ "step": 35500
481
+ },
482
+ {
483
+ "epoch": 5.25,
484
+ "learning_rate": 4.7537161177499274e-05,
485
+ "loss": 0.0752,
486
+ "step": 36000
487
+ },
488
+ {
489
+ "epoch": 5.32,
490
+ "learning_rate": 4.680851063829788e-05,
491
+ "loss": 0.0718,
492
+ "step": 36500
493
+ },
494
+ {
495
+ "epoch": 5.39,
496
+ "learning_rate": 4.6079860099096475e-05,
497
+ "loss": 0.0772,
498
+ "step": 37000
499
+ },
500
+ {
501
+ "epoch": 5.46,
502
+ "learning_rate": 4.535120955989507e-05,
503
+ "loss": 0.0823,
504
+ "step": 37500
505
+ },
506
+ {
507
+ "epoch": 5.54,
508
+ "learning_rate": 4.4622559020693675e-05,
509
+ "loss": 0.075,
510
+ "step": 38000
511
+ },
512
+ {
513
+ "epoch": 5.61,
514
+ "learning_rate": 4.389390848149228e-05,
515
+ "loss": 0.0805,
516
+ "step": 38500
517
+ },
518
+ {
519
+ "epoch": 5.68,
520
+ "learning_rate": 4.316525794229088e-05,
521
+ "loss": 0.0825,
522
+ "step": 39000
523
+ },
524
+ {
525
+ "epoch": 5.76,
526
+ "learning_rate": 4.243660740308948e-05,
527
+ "loss": 0.0799,
528
+ "step": 39500
529
+ },
530
+ {
531
+ "epoch": 5.83,
532
+ "learning_rate": 4.1707956863888084e-05,
533
+ "loss": 0.0868,
534
+ "step": 40000
535
+ },
536
+ {
537
+ "epoch": 5.9,
538
+ "learning_rate": 4.097930632468668e-05,
539
+ "loss": 0.0832,
540
+ "step": 40500
541
+ },
542
+ {
543
+ "epoch": 5.97,
544
+ "learning_rate": 4.025065578548528e-05,
545
+ "loss": 0.0837,
546
+ "step": 41000
547
+ },
548
+ {
549
+ "epoch": 6.0,
550
+ "eval_accuracy": 0.9370614136971593,
551
+ "eval_loss": 0.23802441358566284,
552
+ "eval_runtime": 153.3499,
553
+ "eval_samples_per_second": 29.834,
554
+ "eval_steps_per_second": 4.976,
555
+ "step": 41172
556
+ },
557
+ {
558
+ "epoch": 6.05,
559
+ "learning_rate": 3.952200524628388e-05,
560
+ "loss": 0.0715,
561
+ "step": 41500
562
+ },
563
+ {
564
+ "epoch": 6.12,
565
+ "learning_rate": 3.8793354707082485e-05,
566
+ "loss": 0.0696,
567
+ "step": 42000
568
+ },
569
+ {
570
+ "epoch": 6.19,
571
+ "learning_rate": 3.806470416788109e-05,
572
+ "loss": 0.0647,
573
+ "step": 42500
574
+ },
575
+ {
576
+ "epoch": 6.27,
577
+ "learning_rate": 3.7336053628679686e-05,
578
+ "loss": 0.0662,
579
+ "step": 43000
580
+ },
581
+ {
582
+ "epoch": 6.34,
583
+ "learning_rate": 3.660740308947829e-05,
584
+ "loss": 0.0686,
585
+ "step": 43500
586
+ },
587
+ {
588
+ "epoch": 6.41,
589
+ "learning_rate": 3.587875255027689e-05,
590
+ "loss": 0.0684,
591
+ "step": 44000
592
+ },
593
+ {
594
+ "epoch": 6.48,
595
+ "learning_rate": 3.515010201107549e-05,
596
+ "loss": 0.0666,
597
+ "step": 44500
598
+ },
599
+ {
600
+ "epoch": 6.56,
601
+ "learning_rate": 3.442145147187409e-05,
602
+ "loss": 0.0684,
603
+ "step": 45000
604
+ },
605
+ {
606
+ "epoch": 6.63,
607
+ "learning_rate": 3.369280093267269e-05,
608
+ "loss": 0.0685,
609
+ "step": 45500
610
+ },
611
+ {
612
+ "epoch": 6.7,
613
+ "learning_rate": 3.2964150393471295e-05,
614
+ "loss": 0.0772,
615
+ "step": 46000
616
+ },
617
+ {
618
+ "epoch": 6.78,
619
+ "learning_rate": 3.22354998542699e-05,
620
+ "loss": 0.0746,
621
+ "step": 46500
622
+ },
623
+ {
624
+ "epoch": 6.85,
625
+ "learning_rate": 3.1506849315068496e-05,
626
+ "loss": 0.0707,
627
+ "step": 47000
628
+ },
629
+ {
630
+ "epoch": 6.92,
631
+ "learning_rate": 3.077819877586709e-05,
632
+ "loss": 0.0699,
633
+ "step": 47500
634
+ },
635
+ {
636
+ "epoch": 7.0,
637
+ "learning_rate": 3.0049548236665697e-05,
638
+ "loss": 0.0735,
639
+ "step": 48000
640
+ },
641
+ {
642
+ "epoch": 7.0,
643
+ "eval_accuracy": 0.9368017592492218,
644
+ "eval_loss": 0.2894147038459778,
645
+ "eval_runtime": 153.3728,
646
+ "eval_samples_per_second": 29.829,
647
+ "eval_steps_per_second": 4.975,
648
+ "step": 48034
649
+ },
650
+ {
651
+ "epoch": 7.07,
652
+ "learning_rate": 2.93208976974643e-05,
653
+ "loss": 0.0579,
654
+ "step": 48500
655
+ },
656
+ {
657
+ "epoch": 7.14,
658
+ "learning_rate": 2.8592247158262898e-05,
659
+ "loss": 0.0551,
660
+ "step": 49000
661
+ },
662
+ {
663
+ "epoch": 7.21,
664
+ "learning_rate": 2.7863596619061498e-05,
665
+ "loss": 0.0637,
666
+ "step": 49500
667
+ },
668
+ {
669
+ "epoch": 7.29,
670
+ "learning_rate": 2.7134946079860102e-05,
671
+ "loss": 0.0639,
672
+ "step": 50000
673
+ },
674
+ {
675
+ "epoch": 7.36,
676
+ "learning_rate": 2.64062955406587e-05,
677
+ "loss": 0.0601,
678
+ "step": 50500
679
+ },
680
+ {
681
+ "epoch": 7.43,
682
+ "learning_rate": 2.5677645001457303e-05,
683
+ "loss": 0.0653,
684
+ "step": 51000
685
+ },
686
+ {
687
+ "epoch": 7.51,
688
+ "learning_rate": 2.4948994462255903e-05,
689
+ "loss": 0.0594,
690
+ "step": 51500
691
+ },
692
+ {
693
+ "epoch": 7.58,
694
+ "learning_rate": 2.4220343923054503e-05,
695
+ "loss": 0.0642,
696
+ "step": 52000
697
+ },
698
+ {
699
+ "epoch": 7.65,
700
+ "learning_rate": 2.3491693383853107e-05,
701
+ "loss": 0.0612,
702
+ "step": 52500
703
+ },
704
+ {
705
+ "epoch": 7.72,
706
+ "learning_rate": 2.2763042844651704e-05,
707
+ "loss": 0.0695,
708
+ "step": 53000
709
+ },
710
+ {
711
+ "epoch": 7.8,
712
+ "learning_rate": 2.2034392305450308e-05,
713
+ "loss": 0.0614,
714
+ "step": 53500
715
+ },
716
+ {
717
+ "epoch": 7.87,
718
+ "learning_rate": 2.130574176624891e-05,
719
+ "loss": 0.0634,
720
+ "step": 54000
721
+ },
722
+ {
723
+ "epoch": 7.94,
724
+ "learning_rate": 2.057709122704751e-05,
725
+ "loss": 0.0623,
726
+ "step": 54500
727
+ },
728
+ {
729
+ "epoch": 8.0,
730
+ "eval_accuracy": 0.9354890330840031,
731
+ "eval_loss": 0.30701231956481934,
732
+ "eval_runtime": 153.4064,
733
+ "eval_samples_per_second": 29.823,
734
+ "eval_steps_per_second": 4.974,
735
+ "step": 54896
736
+ },
737
+ {
738
+ "epoch": 8.02,
739
+ "learning_rate": 1.984844068784611e-05,
740
+ "loss": 0.0671,
741
+ "step": 55000
742
+ },
743
+ {
744
+ "epoch": 8.09,
745
+ "learning_rate": 1.9119790148644713e-05,
746
+ "loss": 0.0514,
747
+ "step": 55500
748
+ },
749
+ {
750
+ "epoch": 8.16,
751
+ "learning_rate": 1.839113960944331e-05,
752
+ "loss": 0.0537,
753
+ "step": 56000
754
+ },
755
+ {
756
+ "epoch": 8.23,
757
+ "learning_rate": 1.7662489070241914e-05,
758
+ "loss": 0.0543,
759
+ "step": 56500
760
+ },
761
+ {
762
+ "epoch": 8.31,
763
+ "learning_rate": 1.6933838531040514e-05,
764
+ "loss": 0.0553,
765
+ "step": 57000
766
+ },
767
+ {
768
+ "epoch": 8.38,
769
+ "learning_rate": 1.6205187991839114e-05,
770
+ "loss": 0.0558,
771
+ "step": 57500
772
+ },
773
+ {
774
+ "epoch": 8.45,
775
+ "learning_rate": 1.5476537452637715e-05,
776
+ "loss": 0.0545,
777
+ "step": 58000
778
+ },
779
+ {
780
+ "epoch": 8.53,
781
+ "learning_rate": 1.4747886913436319e-05,
782
+ "loss": 0.0593,
783
+ "step": 58500
784
+ },
785
+ {
786
+ "epoch": 8.6,
787
+ "learning_rate": 1.4019236374234917e-05,
788
+ "loss": 0.0601,
789
+ "step": 59000
790
+ },
791
+ {
792
+ "epoch": 8.67,
793
+ "learning_rate": 1.3290585835033518e-05,
794
+ "loss": 0.053,
795
+ "step": 59500
796
+ },
797
+ {
798
+ "epoch": 8.74,
799
+ "learning_rate": 1.256193529583212e-05,
800
+ "loss": 0.0552,
801
+ "step": 60000
802
+ },
803
+ {
804
+ "epoch": 8.82,
805
+ "learning_rate": 1.183328475663072e-05,
806
+ "loss": 0.0565,
807
+ "step": 60500
808
+ },
809
+ {
810
+ "epoch": 8.89,
811
+ "learning_rate": 1.1104634217429322e-05,
812
+ "loss": 0.0545,
813
+ "step": 61000
814
+ },
815
+ {
816
+ "epoch": 8.96,
817
+ "learning_rate": 1.0375983678227923e-05,
818
+ "loss": 0.0535,
819
+ "step": 61500
820
+ },
821
+ {
822
+ "epoch": 9.0,
823
+ "eval_accuracy": 0.9346114733196021,
824
+ "eval_loss": 0.3566416800022125,
825
+ "eval_runtime": 153.4241,
826
+ "eval_samples_per_second": 29.819,
827
+ "eval_steps_per_second": 4.973,
828
+ "step": 61758
829
+ },
830
+ {
831
+ "epoch": 9.04,
832
+ "learning_rate": 9.647333139026523e-06,
833
+ "loss": 0.0533,
834
+ "step": 62000
835
+ },
836
+ {
837
+ "epoch": 9.11,
838
+ "learning_rate": 8.918682599825125e-06,
839
+ "loss": 0.0496,
840
+ "step": 62500
841
+ },
842
+ {
843
+ "epoch": 9.18,
844
+ "learning_rate": 8.190032060623726e-06,
845
+ "loss": 0.0455,
846
+ "step": 63000
847
+ },
848
+ {
849
+ "epoch": 9.25,
850
+ "learning_rate": 7.461381521422327e-06,
851
+ "loss": 0.047,
852
+ "step": 63500
853
+ },
854
+ {
855
+ "epoch": 9.33,
856
+ "learning_rate": 6.732730982220928e-06,
857
+ "loss": 0.0467,
858
+ "step": 64000
859
+ },
860
+ {
861
+ "epoch": 9.4,
862
+ "learning_rate": 6.0040804430195285e-06,
863
+ "loss": 0.0466,
864
+ "step": 64500
865
+ },
866
+ {
867
+ "epoch": 9.47,
868
+ "learning_rate": 5.275429903818129e-06,
869
+ "loss": 0.0456,
870
+ "step": 65000
871
+ },
872
+ {
873
+ "epoch": 9.55,
874
+ "learning_rate": 4.54677936461673e-06,
875
+ "loss": 0.0516,
876
+ "step": 65500
877
+ },
878
+ {
879
+ "epoch": 9.62,
880
+ "learning_rate": 3.818128825415331e-06,
881
+ "loss": 0.0531,
882
+ "step": 66000
883
+ },
884
+ {
885
+ "epoch": 9.69,
886
+ "learning_rate": 3.0894782862139318e-06,
887
+ "loss": 0.0507,
888
+ "step": 66500
889
+ },
890
+ {
891
+ "epoch": 9.76,
892
+ "learning_rate": 2.360827747012533e-06,
893
+ "loss": 0.0501,
894
+ "step": 67000
895
+ },
896
+ {
897
+ "epoch": 9.84,
898
+ "learning_rate": 1.632177207811134e-06,
899
+ "loss": 0.0505,
900
+ "step": 67500
901
+ },
902
+ {
903
+ "epoch": 9.91,
904
+ "learning_rate": 9.035266686097348e-07,
905
+ "loss": 0.0491,
906
+ "step": 68000
907
+ },
908
+ {
909
+ "epoch": 9.98,
910
+ "learning_rate": 1.7487612940833577e-07,
911
+ "loss": 0.0483,
912
+ "step": 68500
913
+ },
914
+ {
915
+ "epoch": 10.0,
916
+ "eval_accuracy": 0.9332295747964836,
917
+ "eval_loss": 0.41370126605033875,
918
+ "eval_runtime": 153.4718,
919
+ "eval_samples_per_second": 29.81,
920
+ "eval_steps_per_second": 4.972,
921
+ "step": 68620
922
+ }
923
+ ],
924
+ "logging_steps": 500,
925
+ "max_steps": 68620,
926
+ "num_input_tokens_seen": 0,
927
+ "num_train_epochs": 10,
928
+ "save_steps": 500,
929
+ "total_flos": 2.750401151541956e+17,
930
+ "train_batch_size": 6,
931
+ "trial_name": null,
932
+ "trial_params": null
933
+ }
training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5a935ff0efd6ea27ff3615bcbc28258efbb58fae131ee39f17cf7193886cc083
3
+ size 4920
vocab.txt ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <cls>
2
+ <pad>
3
+ <eos>
4
+ <unk>
5
+ L
6
+ A
7
+ G
8
+ V
9
+ S
10
+ E
11
+ R
12
+ T
13
+ I
14
+ D
15
+ P
16
+ K
17
+ Q
18
+ N
19
+ F
20
+ Y
21
+ M
22
+ H
23
+ W
24
+ C
25
+ X
26
+ B
27
+ U
28
+ Z
29
+ O
30
+ .
31
+ -
32
+ <null_1>
33
+ <mask>