scores/deepseek-r1-distill-qwen-7b-iq3_m.mmlu

common_init_from_params: setting dry_penalty_last_n to ctx_size = 768
common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)

system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 |
multiple_choice_score: there are 1548 tasks in prompt
multiple_choice_score: selecting 750 random tasks from 1548 tasks available
multiple_choice_score: preparing task data...done
multiple_choice_score : calculating TruthfulQA score over 750 tasks.

task	acc_norm
1	100.00000000
2	50.00000000
3	33.33333333
4	50.00000000
5	40.00000000
6	33.33333333
7	42.85714286
8	50.00000000
9	44.44444444
10	40.00000000
11	36.36363636
12	33.33333333
13	38.46153846
14	35.71428571
15	40.00000000
16	43.75000000
17	41.17647059
18	38.88888889
19	42.10526316
20	40.00000000
21	38.09523810
22	36.36363636
23	34.78260870
24	33.33333333
25	36.00000000
26	34.61538462
27	33.33333333
28	35.71428571
29	37.93103448
30	40.00000000
31	41.93548387
32	43.75000000
33	42.42424242
34	41.17647059
35	42.85714286
36	41.66666667
37	40.54054054
38	42.10526316
39	41.02564103
40	40.00000000
41	41.46341463
42	40.47619048
43	39.53488372
44	38.63636364
45	37.77777778
46	36.95652174
47	36.17021277
48	37.50000000
49	38.77551020
50	40.00000000
51	39.21568627
52	38.46153846
53	37.73584906
54	37.03703704
55	38.18181818
56	39.28571429
57	38.59649123
58	37.93103448
59	37.28813559
60	36.66666667
61	36.06557377
62	35.48387097
63	34.92063492
64	34.37500000
65	33.84615385
66	33.33333333
67	32.83582090
68	32.35294118
69	33.33333333
70	34.28571429
71	33.80281690
72	34.72222222
73	34.24657534
74	33.78378378
75	34.66666667
76	35.52631579
77	36.36363636
78	37.17948718
79	36.70886076
80	37.50000000
81	37.03703704
82	36.58536585
83	37.34939759
84	38.09523810
85	37.64705882
86	37.20930233
87	37.93103448
88	38.63636364
89	38.20224719
90	37.77777778
91	37.36263736
92	36.95652174
93	36.55913978
94	36.17021277
95	35.78947368
96	36.45833333
97	36.08247423
98	35.71428571
99	35.35353535
100	35.00000000
101	34.65346535
102	34.31372549
103	33.98058252
104	33.65384615
105	33.33333333
106	33.01886792
107	32.71028037
108	33.33333333
109	33.02752294
110	32.72727273
111	32.43243243
112	32.14285714
113	32.74336283
114	33.33333333
115	33.91304348
116	33.62068966
117	33.33333333
118	33.05084746
119	32.77310924
120	32.50000000
121	32.23140496
122	31.96721311
123	31.70731707
124	31.45161290
125	32.00000000
126	31.74603175
127	31.49606299
128	31.25000000
129	31.00775194
130	30.76923077
131	31.29770992
132	31.06060606
133	30.82706767
134	30.59701493
135	30.37037037
136	30.88235294
137	30.65693431
138	30.43478261
139	30.21582734
140	30.00000000
141	29.78723404
142	29.57746479
143	30.06993007
144	30.55555556
145	30.34482759
146	30.13698630
147	29.93197279
148	30.40540541
149	30.20134228
150	30.00000000
151	29.80132450
152	30.26315789
153	30.06535948
154	29.87012987
155	29.67741935
156	29.48717949
157	29.93630573
158	30.37974684
159	30.18867925
160	30.62500000
161	30.43478261
162	30.24691358
163	30.06134969
164	29.87804878
165	30.30303030
166	30.12048193
167	30.53892216
168	30.35714286
169	30.17751479
170	30.00000000
171	29.82456140
172	29.65116279
173	29.47976879
174	29.31034483
175	29.14285714
176	29.54545455
177	29.37853107
178	29.21348315
179	29.60893855
180	30.00000000
181	29.83425414
182	29.67032967
183	30.05464481
184	29.89130435
185	30.27027027
186	30.64516129
187	30.48128342
188	30.31914894
189	30.15873016
190	30.00000000
191	29.84293194
192	29.68750000
193	29.53367876
194	29.89690722
195	30.25641026
196	30.61224490
197	30.45685279
198	30.30303030
199	30.15075377
200	30.00000000
201	30.34825871
202	30.19801980
203	30.54187192
204	30.39215686
205	30.73170732
206	30.58252427
207	30.43478261
208	30.28846154
209	30.14354067
210	30.00000000
211	30.33175355
212	30.18867925
213	30.04694836
214	29.90654206
215	29.76744186
216	29.62962963
217	29.49308756
218	29.35779817
219	29.22374429
220	29.09090909
221	28.95927602
222	29.27927928
223	29.14798206
224	29.01785714
225	29.33333333
226	29.20353982
227	29.51541850
228	29.82456140
229	29.69432314
230	29.56521739
231	29.43722944
232	29.31034483
233	29.18454936
234	29.48717949
235	29.36170213
236	29.23728814
237	29.11392405
238	28.99159664
239	28.87029289
240	28.75000000
241	28.63070539
242	28.92561983
243	29.21810700
244	29.09836066
245	29.38775510
246	29.26829268
247	29.14979757
248	29.03225806
249	28.91566265
250	28.80000000
251	28.68525896
252	28.57142857
253	28.85375494
254	28.74015748
255	29.01960784
256	28.90625000
257	29.18287938
258	29.06976744
259	29.34362934
260	29.23076923
261	29.11877395
262	29.00763359
263	28.89733840
264	29.16666667
265	29.05660377
266	28.94736842
267	28.83895131
268	28.73134328
269	28.62453532
270	28.51851852
271	28.41328413
272	28.67647059
273	28.57142857
274	28.46715328
275	28.36363636
276	28.62318841
277	28.88086643
278	28.77697842
279	28.67383513
280	28.57142857
281	28.46975089
282	28.36879433
283	28.26855124
284	28.16901408
285	28.07017544
286	27.97202797
287	28.22299652
288	28.12500000
289	28.02768166
290	28.27586207
291	28.52233677
292	28.76712329
293	29.01023891
294	29.25170068
295	29.15254237
296	29.05405405
297	28.95622896
298	29.19463087
299	29.09698997
300	29.33333333
301	29.23588040
302	29.47019868
303	29.37293729
304	29.60526316
305	29.83606557
306	29.73856209
307	29.64169381
308	29.54545455
309	29.44983819
310	29.35483871
311	29.26045016
312	29.48717949
313	29.71246006
314	29.93630573
315	29.84126984
316	29.74683544
317	29.65299685
318	29.55974843
319	29.78056426
320	29.68750000
321	29.59501558
322	29.50310559
323	29.41176471
324	29.32098765
325	29.53846154
326	29.44785276
327	29.66360856
328	29.87804878
329	29.78723404
330	29.69696970
331	29.60725076
332	29.81927711
333	29.72972973
334	29.64071856
335	29.85074627
336	29.76190476
337	29.67359050
338	29.58579882
339	29.49852507
340	29.41176471
341	29.32551320
342	29.23976608
343	29.15451895
344	29.06976744
345	28.98550725
346	29.19075145
347	29.39481268
348	29.31034483
349	29.51289398
350	29.71428571
351	29.62962963
352	29.54545455
353	29.74504249
354	29.66101695
355	29.57746479
356	29.77528090
357	29.69187675
358	29.60893855
359	29.52646240
360	29.72222222
361	29.91689751
362	29.83425414
363	30.02754821
364	29.94505495
365	29.86301370
366	29.78142077
367	29.70027248
368	29.61956522
369	29.53929539
370	29.72972973
371	29.64959569
372	29.56989247
373	29.49061662
374	29.41176471
375	29.60000000
376	29.52127660
377	29.44297082
378	29.36507937
379	29.28759894
380	29.21052632
381	29.13385827
382	29.05759162
383	29.24281984
384	29.16666667
385	29.09090909
386	29.01554404
387	28.94056848
388	28.86597938
389	29.04884319
390	28.97435897
391	29.15601023
392	29.33673469
393	29.26208651
394	29.18781726
395	29.11392405
396	29.29292929
397	29.21914358
398	29.14572864
399	29.32330827
400	29.25000000
401	29.42643392
402	29.35323383
403	29.28039702
404	29.20792079
405	29.13580247
406	29.06403941
407	28.99262899
408	28.92156863
409	29.09535452
410	29.02439024
411	28.95377129
412	28.88349515
413	29.05569007
414	29.22705314
415	29.39759036
416	29.32692308
417	29.25659472
418	29.18660287
419	29.35560859
420	29.52380952
421	29.45368171
422	29.38388626
423	29.31442080
424	29.48113208
425	29.64705882
426	29.57746479
427	29.50819672
428	29.67289720
429	29.83682984
430	29.76744186
431	29.93039443
432	30.09259259
433	30.02309469
434	30.18433180
435	30.11494253
436	30.04587156
437	29.97711670
438	30.13698630
439	30.29612756
440	30.22727273
441	30.15873016
442	30.09049774
443	30.02257336
444	29.95495495
445	29.88764045
446	29.82062780
447	29.75391499
448	29.68750000
449	29.62138085
450	29.55555556
451	29.49002217
452	29.64601770
453	29.80132450
454	29.73568282
455	29.67032967
456	29.82456140
457	29.75929978
458	29.69432314
459	29.84749455
460	30.00000000
461	29.93492408
462	29.87012987
463	30.02159827
464	29.95689655
465	29.89247312
466	29.82832618
467	29.97858672
468	29.91452991
469	30.06396588
470	30.00000000
471	29.93630573
472	29.87288136
473	30.02114165
474	29.95780591
475	29.89473684
476	30.04201681
477	29.97903564
478	30.12552301
479	30.27139875
480	30.20833333
481	30.14553015
482	30.08298755
483	30.02070393
484	29.95867769
485	29.89690722
486	29.83539095
487	29.97946612
488	29.91803279
489	29.85685072
490	30.00000000
491	30.14256619
492	30.08130081
493	30.02028398
494	30.16194332
495	30.30303030
496	30.44354839
497	30.38229376
498	30.32128514
499	30.26052104
500	30.40000000
501	30.33932136
502	30.27888446
503	30.21868787
504	30.35714286
505	30.29702970
506	30.43478261
507	30.37475345
508	30.31496063
509	30.25540275
510	30.39215686
511	30.33268102
512	30.27343750
513	30.21442495
514	30.15564202
515	30.09708738
516	30.03875969
517	30.17408124
518	30.30888031
519	30.25048170
520	30.19230769
521	30.13435701
522	30.26819923
523	30.40152964
524	30.53435115
525	30.47619048
526	30.41825095
527	30.55028463
528	30.49242424
529	30.62381853
530	30.75471698
531	30.69679849
532	30.63909774
533	30.58161351
534	30.52434457
535	30.65420561
536	30.59701493
537	30.72625698
538	30.85501859
539	30.79777365
540	30.74074074
541	30.86876155
542	30.99630996
543	30.93922652
544	30.88235294
545	30.82568807
546	30.76923077
547	30.71297989
548	30.83941606
549	30.78324226
550	30.72727273
551	30.67150635
552	30.79710145
553	30.74141049
554	30.68592058
555	30.63063063
556	30.57553957
557	30.52064632
558	30.64516129
559	30.59033989
560	30.53571429
561	30.48128342
562	30.42704626
563	30.55062167
564	30.49645390
565	30.44247788
566	30.38869258
567	30.33509700
568	30.45774648
569	30.40421793
570	30.35087719
571	30.47285464
572	30.41958042
573	30.36649215
574	30.48780488
575	30.43478261
576	30.55555556
577	30.67590988
578	30.62283737
579	30.56994819
580	30.51724138
581	30.46471601
582	30.41237113
583	30.36020583
584	30.47945205
585	30.42735043
586	30.37542662
587	30.32367973
588	30.27210884
589	30.22071307
590	30.16949153
591	30.11844332
592	30.06756757
593	30.01686341
594	30.13468013
595	30.08403361
596	30.20134228
597	30.15075377
598	30.10033445
599	30.21702838
600	30.16666667
601	30.28286190
602	30.23255814
603	30.34825871
604	30.29801325
605	30.41322314
606	30.36303630
607	30.47775947
608	30.42763158
609	30.37766831
610	30.32786885
611	30.27823241
612	30.22875817
613	30.17944535
614	30.29315961
615	30.24390244
616	30.35714286
617	30.47001621
618	30.42071197
619	30.37156704
620	30.32258065
621	30.27375201
622	30.38585209
623	30.33707865
624	30.28846154
625	30.24000000
626	30.35143770
627	30.30303030
628	30.41401274
629	30.52464229
630	30.47619048
631	30.42789223
632	30.53797468
633	30.48973144
634	30.44164038
635	30.39370079
636	30.34591195
637	30.29827316
638	30.25078370
639	30.35993740
640	30.46875000
641	30.42121685
642	30.52959502
643	30.48211509
644	30.59006211
645	30.54263566
646	30.49535604
647	30.44822257
648	30.40123457
649	30.35439137
650	30.30769231
651	30.26113671
652	30.21472393
653	30.16845329
654	30.27522936
655	30.22900763
656	30.33536585
657	30.28919330
658	30.39513678
659	30.34901366
660	30.30303030
661	30.25718608
662	30.21148036
663	30.16591252
664	30.12048193
665	30.07518797
666	30.03003003
667	29.98500750
668	30.08982036
669	30.04484305
670	30.14925373
671	30.10432191
672	30.05952381
673	30.01485884
674	29.97032641
675	29.92592593
676	29.88165680
677	29.83751846
678	29.79351032
679	29.74963181
680	29.85294118
681	29.95594714
682	29.91202346
683	30.01464129
684	29.97076023
685	29.92700730
686	29.88338192
687	29.83988355
688	29.79651163
689	29.75326560
690	29.71014493
691	29.81186686
692	29.76878613
693	29.72582973
694	29.68299712
695	29.64028777
696	29.74137931
697	29.69870875
698	29.65616046
699	29.75679542
700	29.71428571
701	29.81455064
702	29.91452991
703	30.01422475
704	29.97159091
705	29.92907801
706	29.88668555
707	29.84441301
708	29.80225989
709	29.76022567
710	29.85915493
711	29.95780591
712	29.91573034
713	30.01402525
714	30.11204482
715	30.06993007
716	30.02793296
717	29.98605300
718	29.94428969
719	29.90264256
720	29.86111111
721	29.81969487
722	29.77839335
723	29.87551867
724	29.83425414
725	29.93103448
726	30.02754821
727	29.98624484
728	29.94505495
729	30.04115226
730	30.00000000
731	29.95896033
732	30.05464481
733	30.01364256
734	29.97275204
735	29.93197279
736	30.02717391
737	30.12211669
738	30.08130081
739	30.17591340
740	30.27027027
741	30.22941970
742	30.32345013
743	30.41722746
744	30.51075269
745	30.46979866
746	30.42895442
747	30.38821954
748	30.34759358
749	30.30707610
750	30.40000000

Final result: 30.4000 ±1.6807
Random chance: 25.0000 ±1.5822