scores/deepseek-r1-distill-qwen-7b-iq3_s.mmlu

common_init_from_params: setting dry_penalty_last_n to ctx_size = 768
common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)

system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 |
multiple_choice_score: there are 1548 tasks in prompt
multiple_choice_score: selecting 750 random tasks from 1548 tasks available
multiple_choice_score: preparing task data...done
multiple_choice_score : calculating TruthfulQA score over 750 tasks.

task	acc_norm
1	100.00000000
2	50.00000000
3	33.33333333
4	50.00000000
5	40.00000000
6	33.33333333
7	42.85714286
8	50.00000000
9	44.44444444
10	40.00000000
11	36.36363636
12	33.33333333
13	38.46153846
14	35.71428571
15	33.33333333
16	37.50000000
17	35.29411765
18	33.33333333
19	36.84210526
20	35.00000000
21	33.33333333
22	31.81818182
23	30.43478261
24	29.16666667
25	32.00000000
26	30.76923077
27	29.62962963
28	32.14285714
29	34.48275862
30	36.66666667
31	38.70967742
32	40.62500000
33	39.39393939
34	38.23529412
35	40.00000000
36	38.88888889
37	37.83783784
38	39.47368421
39	38.46153846
40	37.50000000
41	36.58536585
42	35.71428571
43	34.88372093
44	34.09090909
45	33.33333333
46	32.60869565
47	31.91489362
48	33.33333333
49	34.69387755
50	36.00000000
51	35.29411765
52	34.61538462
53	33.96226415
54	33.33333333
55	34.54545455
56	35.71428571
57	35.08771930
58	34.48275862
59	33.89830508
60	33.33333333
61	32.78688525
62	32.25806452
63	31.74603175
64	31.25000000
65	30.76923077
66	31.81818182
67	31.34328358
68	30.88235294
69	31.88405797
70	32.85714286
71	32.39436620
72	33.33333333
73	32.87671233
74	32.43243243
75	33.33333333
76	34.21052632
77	35.06493506
78	35.89743590
79	35.44303797
80	36.25000000
81	35.80246914
82	35.36585366
83	36.14457831
84	36.90476190
85	36.47058824
86	36.04651163
87	36.78160920
88	37.50000000
89	37.07865169
90	36.66666667
91	36.26373626
92	35.86956522
93	35.48387097
94	35.10638298
95	34.73684211
96	35.41666667
97	35.05154639
98	34.69387755
99	34.34343434
100	34.00000000
101	33.66336634
102	33.33333333
103	33.00970874
104	32.69230769
105	32.38095238
106	32.07547170
107	31.77570093
108	32.40740741
109	32.11009174
110	31.81818182
111	31.53153153
112	31.25000000
113	31.85840708
114	32.45614035
115	33.04347826
116	32.75862069
117	32.47863248
118	32.20338983
119	31.93277311
120	31.66666667
121	31.40495868
122	31.14754098
123	30.89430894
124	30.64516129
125	31.20000000
126	30.95238095
127	30.70866142
128	30.46875000
129	30.23255814
130	30.00000000
131	30.53435115
132	30.30303030
133	30.07518797
134	29.85074627
135	29.62962963
136	30.14705882
137	29.92700730
138	29.71014493
139	29.49640288
140	29.28571429
141	29.07801418
142	29.57746479
143	30.06993007
144	30.55555556
145	31.03448276
146	30.82191781
147	30.61224490
148	31.08108108
149	30.87248322
150	30.66666667
151	30.46357616
152	30.92105263
153	30.71895425
154	31.16883117
155	30.96774194
156	30.76923077
157	31.21019108
158	31.64556962
159	31.44654088
160	31.87500000
161	31.67701863
162	31.48148148
163	31.28834356
164	31.09756098
165	31.51515152
166	31.32530120
167	31.73652695
168	31.54761905
169	31.36094675
170	31.17647059
171	30.99415205
172	30.81395349
173	30.63583815
174	30.45977011
175	30.28571429
176	30.68181818
177	30.50847458
178	30.33707865
179	30.72625698
180	30.55555556
181	30.38674033
182	30.21978022
183	30.60109290
184	30.43478261
185	30.81081081
186	31.18279570
187	31.01604278
188	30.85106383
189	30.68783069
190	30.52631579
191	30.36649215
192	30.20833333
193	30.05181347
194	30.41237113
195	30.76923077
196	31.12244898
197	30.96446701
198	30.80808081
199	30.65326633
200	30.50000000
201	30.84577114
202	30.69306931
203	31.03448276
204	30.88235294
205	31.21951220
206	31.06796117
207	30.91787440
208	30.76923077
209	30.62200957
210	30.47619048
211	30.80568720
212	30.66037736
213	30.51643192
214	30.37383178
215	30.23255814
216	30.09259259
217	29.95391705
218	29.81651376
219	29.68036530
220	29.54545455
221	29.41176471
222	29.72972973
223	29.59641256
224	29.91071429
225	30.22222222
226	30.08849558
227	30.39647577
228	30.70175439
229	30.56768559
230	30.43478261
231	30.30303030
232	30.17241379
233	30.04291845
234	30.34188034
235	30.21276596
236	30.08474576
237	29.95780591
238	29.83193277
239	29.70711297
240	29.58333333
241	29.46058091
242	29.75206612
243	30.04115226
244	29.91803279
245	30.20408163
246	30.08130081
247	29.95951417
248	29.83870968
249	29.71887550
250	29.60000000
251	29.48207171
252	29.36507937
253	29.64426877
254	29.52755906
255	29.80392157
256	29.68750000
257	29.96108949
258	29.84496124
259	30.11583012
260	30.00000000
261	29.88505747
262	29.77099237
263	29.65779468
264	29.92424242
265	30.18867925
266	30.07518797
267	29.96254682
268	29.85074627
269	29.73977695
270	29.62962963
271	29.52029520
272	29.77941176
273	29.67032967
274	29.56204380
275	29.45454545
276	29.71014493
277	29.96389892
278	30.21582734
279	30.10752688
280	30.00000000
281	29.89323843
282	29.78723404
283	29.68197880
284	29.57746479
285	29.47368421
286	29.37062937
287	29.61672474
288	29.51388889
289	29.41176471
290	29.65517241
291	29.89690722
292	30.13698630
293	30.37542662
294	30.61224490
295	30.50847458
296	30.40540541
297	30.30303030
298	30.53691275
299	30.43478261
300	30.66666667
301	30.56478405
302	30.79470199
303	30.69306931
304	30.92105263
305	31.14754098
306	31.04575163
307	31.27035831
308	31.16883117
309	31.06796117
310	30.96774194
311	30.86816720
312	31.08974359
313	31.30990415
314	31.52866242
315	31.42857143
316	31.32911392
317	31.23028391
318	31.13207547
319	31.34796238
320	31.25000000
321	31.15264798
322	31.05590062
323	30.95975232
324	30.86419753
325	31.07692308
326	30.98159509
327	31.19266055
328	31.09756098
329	31.00303951
330	30.90909091
331	30.81570997
332	31.02409639
333	30.93093093
334	30.83832335
335	31.04477612
336	30.95238095
337	30.86053412
338	30.76923077
339	30.67846608
340	30.58823529
341	30.49853372
342	30.40935673
343	30.32069971
344	30.23255814
345	30.14492754
346	30.34682081
347	30.54755043
348	30.45977011
349	30.65902579
350	30.85714286
351	30.76923077
352	30.68181818
353	30.87818697
354	30.79096045
355	30.70422535
356	30.89887640
357	30.81232493
358	30.72625698
359	30.91922006
360	31.11111111
361	31.30193906
362	31.21546961
363	31.40495868
364	31.31868132
365	31.23287671
366	31.14754098
367	31.06267030
368	30.97826087
369	30.89430894
370	31.08108108
371	30.99730458
372	30.91397849
373	30.83109920
374	31.01604278
375	30.93333333
376	30.85106383
377	30.76923077
378	30.68783069
379	30.60686016
380	30.52631579
381	30.70866142
382	30.62827225
383	30.80939948
384	30.72916667
385	30.64935065
386	30.56994819
387	30.49095607
388	30.41237113
389	30.59125964
390	30.51282051
391	30.69053708
392	30.86734694
393	30.78880407
394	30.71065990
395	30.63291139
396	30.80808081
397	30.73047859
398	30.65326633
399	30.82706767
400	30.75000000
401	30.92269327
402	30.84577114
403	30.76923077
404	30.69306931
405	30.61728395
406	30.78817734
407	30.71253071
408	30.63725490
409	30.80684597
410	30.73170732
411	30.65693431
412	30.58252427
413	30.75060533
414	30.91787440
415	31.08433735
416	31.00961538
417	30.93525180
418	30.86124402
419	31.02625298
420	31.19047619
421	31.11638955
422	31.04265403
423	30.96926714
424	31.13207547
425	31.29411765
426	31.22065728
427	31.14754098
428	31.07476636
429	31.23543124
430	31.16279070
431	31.32250580
432	31.48148148
433	31.40877598
434	31.56682028
435	31.49425287
436	31.42201835
437	31.35011442
438	31.50684932
439	31.66287016
440	31.59090909
441	31.51927438
442	31.44796380
443	31.37697517
444	31.30630631
445	31.23595506
446	31.16591928
447	31.09619687
448	31.02678571
449	30.95768374
450	30.88888889
451	30.82039911
452	30.97345133
453	31.12582781
454	31.05726872
455	30.98901099
456	31.14035088
457	31.07221007
458	31.00436681
459	31.15468410
460	31.30434783
461	31.23644252
462	31.16883117
463	31.31749460
464	31.25000000
465	31.18279570
466	31.11587983
467	31.26338330
468	31.19658120
469	31.34328358
470	31.27659574
471	31.21019108
472	31.14406780
473	31.28964059
474	31.22362869
475	31.15789474
476	31.30252101
477	31.23689727
478	31.38075314
479	31.52400835
480	31.45833333
481	31.39293139
482	31.32780083
483	31.26293996
484	31.19834711
485	31.13402062
486	31.06995885
487	31.21149897
488	31.14754098
489	31.08384458
490	31.22448980
491	31.36456212
492	31.30081301
493	31.23732252
494	31.37651822
495	31.51515152
496	31.65322581
497	31.58953722
498	31.52610442
499	31.46292585
500	31.60000000
501	31.53692615
502	31.47410359
503	31.61033797
504	31.54761905
505	31.48514851
506	31.62055336
507	31.55818540
508	31.49606299
509	31.43418468
510	31.56862745
511	31.50684932
512	31.44531250
513	31.38401559
514	31.32295720
515	31.26213592
516	31.20155039
517	31.33462282
518	31.46718147
519	31.40655106
520	31.34615385
521	31.28598848
522	31.41762452
523	31.54875717
524	31.67938931
525	31.61904762
526	31.55893536
527	31.68880455
528	31.62878788
529	31.56899811
530	31.69811321
531	31.63841808
532	31.57894737
533	31.51969981
534	31.46067416
535	31.40186916
536	31.34328358
537	31.47113594
538	31.59851301
539	31.53988868
540	31.48148148
541	31.60813309
542	31.73431734
543	31.86003683
544	31.80147059
545	31.74311927
546	31.68498168
547	31.62705667
548	31.75182482
549	31.69398907
550	31.63636364
551	31.57894737
552	31.70289855
553	31.64556962
554	31.58844765
555	31.53153153
556	31.47482014
557	31.41831239
558	31.54121864
559	31.48479428
560	31.42857143
561	31.37254902
562	31.31672598
563	31.43872114
564	31.38297872
565	31.32743363
566	31.27208481
567	31.21693122
568	31.33802817
569	31.28295255
570	31.22807018
571	31.34851138
572	31.29370629
573	31.23909250
574	31.35888502
575	31.30434783
576	31.42361111
577	31.54246101
578	31.48788927
579	31.43350604
580	31.37931034
581	31.32530120
582	31.27147766
583	31.21783877
584	31.33561644
585	31.28205128
586	31.22866894
587	31.17546848
588	31.12244898
589	31.06960951
590	31.01694915
591	30.96446701
592	30.91216216
593	30.86003373
594	30.97643098
595	30.92436975
596	31.04026846
597	30.98827471
598	30.93645485
599	31.05175292
600	31.00000000
601	31.11480865
602	31.06312292
603	31.17744610
604	31.12582781
605	31.23966942
606	31.18811881
607	31.30148270
608	31.25000000
609	31.19868637
610	31.14754098
611	31.09656301
612	31.04575163
613	30.99510604
614	31.10749186
615	31.05691057
616	31.16883117
617	31.28038898
618	31.22977346
619	31.17932149
620	31.12903226
621	31.07890499
622	31.18971061
623	31.13964687
624	31.08974359
625	31.04000000
626	31.15015974
627	31.10047847
628	31.05095541
629	31.16057234
630	31.11111111
631	31.06180666
632	31.17088608
633	31.12164297
634	31.07255521
635	31.02362205
636	30.97484277
637	30.92621664
638	30.87774295
639	30.82942097
640	30.93750000
641	30.88923557
642	30.99688474
643	30.94867807
644	31.05590062
645	31.00775194
646	30.95975232
647	30.91190108
648	30.86419753
649	30.81664099
650	30.76923077
651	30.72196621
652	30.67484663
653	30.62787136
654	30.73394495
655	30.68702290
656	30.79268293
657	30.74581431
658	30.85106383
659	30.80424886
660	30.75757576
661	30.71104387
662	30.66465257
663	30.61840121
664	30.57228916
665	30.52631579
666	30.48048048
667	30.58470765
668	30.68862275
669	30.64275037
670	30.74626866
671	30.70044709
672	30.65476190
673	30.60921248
674	30.56379822
675	30.51851852
676	30.47337278
677	30.42836041
678	30.38348083
679	30.33873343
680	30.44117647
681	30.54331865
682	30.49853372
683	30.60029283
684	30.55555556
685	30.51094891
686	30.46647230
687	30.42212518
688	30.37790698
689	30.33381713
690	30.28985507
691	30.39073806
692	30.34682081
693	30.30303030
694	30.40345821
695	30.35971223
696	30.45977011
697	30.41606887
698	30.37249284
699	30.47210300
700	30.42857143
701	30.52781740
702	30.62678063
703	30.72546230
704	30.68181818
705	30.63829787
706	30.59490085
707	30.55162659
708	30.50847458
709	30.46544429
710	30.56338028
711	30.66104079
712	30.61797753
713	30.71528752
714	30.81232493
715	30.76923077
716	30.72625698
717	30.68340307
718	30.64066852
719	30.59805285
720	30.55555556
721	30.51317614
722	30.47091413
723	30.56708160
724	30.52486188
725	30.62068966
726	30.71625344
727	30.67400275
728	30.63186813
729	30.72702332
730	30.68493151
731	30.64295486
732	30.73770492
733	30.83219645
734	30.79019074
735	30.74829932
736	30.70652174
737	30.80054274
738	30.75880759
739	30.85250338
740	30.94594595
741	30.90418354
742	30.99730458
743	31.09017497
744	31.18279570
745	31.14093960
746	31.09919571
747	31.05756359
748	31.01604278
749	31.10814419
750	31.20000000

Final result: 31.2000 ±1.6929
Random chance: 25.0000 ±1.5822