scores/deepseek-r1-distill-qwen-7b-q3_k_l.mmlu

common_init_from_params: setting dry_penalty_last_n to ctx_size = 768
common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)

system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 |
multiple_choice_score: there are 1548 tasks in prompt
multiple_choice_score: selecting 750 random tasks from 1548 tasks available
multiple_choice_score: preparing task data...done
multiple_choice_score : calculating TruthfulQA score over 750 tasks.

task	acc_norm
1	0.00000000
2	0.00000000
3	0.00000000
4	25.00000000
5	20.00000000
6	16.66666667
7	28.57142857
8	25.00000000
9	22.22222222
10	20.00000000
11	18.18181818
12	16.66666667
13	23.07692308
14	21.42857143
15	26.66666667
16	31.25000000
17	29.41176471
18	27.77777778
19	31.57894737
20	30.00000000
21	28.57142857
22	27.27272727
23	26.08695652
24	25.00000000
25	28.00000000
26	26.92307692
27	25.92592593
28	28.57142857
29	31.03448276
30	33.33333333
31	35.48387097
32	37.50000000
33	36.36363636
34	35.29411765
35	37.14285714
36	36.11111111
37	35.13513514
38	36.84210526
39	35.89743590
40	35.00000000
41	36.58536585
42	35.71428571
43	34.88372093
44	34.09090909
45	33.33333333
46	32.60869565
47	31.91489362
48	33.33333333
49	34.69387755
50	36.00000000
51	35.29411765
52	36.53846154
53	35.84905660
54	35.18518519
55	36.36363636
56	37.50000000
57	36.84210526
58	36.20689655
59	35.59322034
60	35.00000000
61	34.42622951
62	33.87096774
63	33.33333333
64	32.81250000
65	32.30769231
66	33.33333333
67	32.83582090
68	32.35294118
69	31.88405797
70	32.85714286
71	33.80281690
72	34.72222222
73	34.24657534
74	35.13513514
75	36.00000000
76	36.84210526
77	37.66233766
78	38.46153846
79	37.97468354
80	38.75000000
81	38.27160494
82	37.80487805
83	38.55421687
84	39.28571429
85	38.82352941
86	38.37209302
87	39.08045977
88	39.77272727
89	39.32584270
90	38.88888889
91	38.46153846
92	38.04347826
93	37.63440860
94	37.23404255
95	36.84210526
96	37.50000000
97	37.11340206
98	36.73469388
99	36.36363636
100	37.00000000
101	36.63366337
102	36.27450980
103	35.92233010
104	35.57692308
105	35.23809524
106	34.90566038
107	34.57943925
108	34.25925926
109	33.94495413
110	33.63636364
111	33.33333333
112	33.03571429
113	33.62831858
114	34.21052632
115	34.78260870
116	34.48275862
117	34.18803419
118	33.89830508
119	33.61344538
120	33.33333333
121	33.05785124
122	32.78688525
123	32.52032520
124	32.25806452
125	32.80000000
126	32.53968254
127	32.28346457
128	32.03125000
129	31.78294574
130	31.53846154
131	32.06106870
132	31.81818182
133	31.57894737
134	31.34328358
135	31.11111111
136	31.61764706
137	31.38686131
138	31.15942029
139	31.65467626
140	31.42857143
141	31.20567376
142	31.69014085
143	32.16783217
144	32.63888889
145	32.41379310
146	32.19178082
147	31.97278912
148	32.43243243
149	32.21476510
150	32.00000000
151	31.78807947
152	32.23684211
153	32.67973856
154	32.46753247
155	32.25806452
156	32.05128205
157	32.48407643
158	32.91139241
159	32.70440252
160	33.12500000
161	32.91925466
162	32.71604938
163	32.51533742
164	32.31707317
165	32.72727273
166	32.53012048
167	32.93413174
168	32.73809524
169	32.54437870
170	32.35294118
171	32.16374269
172	31.97674419
173	31.79190751
174	31.60919540
175	31.42857143
176	31.25000000
177	31.07344633
178	30.89887640
179	31.28491620
180	31.11111111
181	30.93922652
182	30.76923077
183	31.14754098
184	30.97826087
185	31.35135135
186	31.72043011
187	31.55080214
188	31.38297872
189	31.21693122
190	31.05263158
191	31.41361257
192	31.25000000
193	31.08808290
194	31.44329897
195	31.79487179
196	31.63265306
197	31.47208122
198	31.31313131
199	31.15577889
200	31.00000000
201	31.34328358
202	31.18811881
203	31.03448276
204	30.88235294
205	31.21951220
206	31.06796117
207	30.91787440
208	30.76923077
209	30.62200957
210	30.47619048
211	30.80568720
212	30.66037736
213	30.51643192
214	30.37383178
215	30.23255814
216	30.09259259
217	29.95391705
218	29.81651376
219	29.68036530
220	29.54545455
221	29.41176471
222	29.72972973
223	29.59641256
224	29.46428571
225	29.77777778
226	29.64601770
227	29.95594714
228	30.26315789
229	30.13100437
230	30.00000000
231	29.87012987
232	29.74137931
233	29.61373391
234	29.91452991
235	29.78723404
236	29.66101695
237	29.53586498
238	29.41176471
239	29.28870293
240	29.58333333
241	29.46058091
242	29.75206612
243	30.04115226
244	29.91803279
245	30.20408163
246	30.08130081
247	29.95951417
248	30.24193548
249	30.12048193
250	30.00000000
251	29.88047809
252	29.76190476
253	29.64426877
254	29.52755906
255	29.80392157
256	30.07812500
257	30.35019455
258	30.23255814
259	30.50193050
260	30.38461538
261	30.26819923
262	30.15267176
263	30.03802281
264	29.92424242
265	29.81132075
266	29.69924812
267	29.58801498
268	29.47761194
269	29.36802974
270	29.25925926
271	29.15129151
272	29.41176471
273	29.30402930
274	29.19708029
275	29.09090909
276	29.34782609
277	29.60288809
278	29.85611511
279	29.74910394
280	30.00000000
281	29.89323843
282	29.78723404
283	29.68197880
284	29.57746479
285	29.47368421
286	29.37062937
287	29.61672474
288	29.51388889
289	29.41176471
290	29.65517241
291	29.89690722
292	30.13698630
293	30.37542662
294	30.61224490
295	30.50847458
296	30.40540541
297	30.30303030
298	30.53691275
299	30.43478261
300	30.66666667
301	30.89700997
302	31.12582781
303	31.02310231
304	31.25000000
305	31.47540984
306	31.37254902
307	31.59609121
308	31.49350649
309	31.71521036
310	31.61290323
311	31.83279743
312	32.05128205
313	32.26837061
314	32.16560510
315	32.06349206
316	31.96202532
317	31.86119874
318	31.76100629
319	31.97492163
320	31.87500000
321	31.77570093
322	31.67701863
323	31.57894737
324	31.48148148
325	31.69230769
326	31.59509202
327	31.80428135
328	31.70731707
329	31.61094225
330	31.51515152
331	31.41993958
332	31.62650602
333	31.53153153
334	31.43712575
335	31.64179104
336	31.54761905
337	31.45400593
338	31.36094675
339	31.26843658
340	31.17647059
341	31.37829912
342	31.28654971
343	31.19533528
344	31.10465116
345	31.01449275
346	31.21387283
347	31.41210375
348	31.32183908
349	31.23209169
350	31.42857143
351	31.33903134
352	31.25000000
353	31.44475921
354	31.35593220
355	31.26760563
356	31.46067416
357	31.37254902
358	31.28491620
359	31.19777159
360	31.38888889
361	31.57894737
362	31.76795580
363	31.95592287
364	31.86813187
365	31.78082192
366	31.69398907
367	31.60762943
368	31.52173913
369	31.43631436
370	31.62162162
371	31.53638814
372	31.45161290
373	31.36729223
374	31.55080214
375	31.73333333
376	31.64893617
377	31.56498674
378	31.48148148
379	31.39841689
380	31.31578947
381	31.49606299
382	31.41361257
383	31.59268930
384	31.51041667
385	31.68831169
386	31.60621762
387	31.52454780
388	31.44329897
389	31.61953728
390	31.79487179
391	31.96930946
392	32.14285714
393	32.06106870
394	31.97969543
395	31.89873418
396	32.07070707
397	31.98992443
398	31.90954774
399	32.08020050
400	32.00000000
401	32.16957606
402	32.08955224
403	32.00992556
404	31.93069307
405	31.85185185
406	31.77339901
407	31.69533170
408	31.61764706
409	31.78484108
410	31.70731707
411	31.63017032
412	31.55339806
413	31.71912833
414	31.88405797
415	32.04819277
416	31.97115385
417	31.89448441
418	31.81818182
419	31.98090692
420	32.14285714
421	32.06650831
422	31.99052133
423	31.91489362
424	32.07547170
425	32.00000000
426	31.92488263
427	31.85011710
428	31.77570093
429	31.93473193
430	31.86046512
431	32.01856148
432	32.17592593
433	32.10161663
434	32.25806452
435	32.18390805
436	32.11009174
437	32.03661327
438	32.19178082
439	32.11845103
440	32.04545455
441	31.97278912
442	31.90045249
443	31.82844244
444	31.75675676
445	31.68539326
446	31.61434978
447	31.54362416
448	31.47321429
449	31.40311804
450	31.33333333
451	31.26385809
452	31.41592920
453	31.56732892
454	31.49779736
455	31.42857143
456	31.57894737
457	31.50984683
458	31.44104803
459	31.59041394
460	31.73913043
461	31.67028200
462	31.60173160
463	31.53347732
464	31.46551724
465	31.39784946
466	31.33047210
467	31.47751606
468	31.41025641
469	31.55650320
470	31.70212766
471	31.63481953
472	31.56779661
473	31.71247357
474	31.64556962
475	31.57894737
476	31.72268908
477	31.65618449
478	31.79916318
479	31.94154489
480	31.87500000
481	31.80873181
482	31.74273859
483	31.67701863
484	31.61157025
485	31.54639175
486	31.48148148
487	31.62217659
488	31.55737705
489	31.49284254
490	31.63265306
491	31.77189409
492	31.70731707
493	31.64300203
494	31.78137652
495	31.91919192
496	32.05645161
497	31.99195171
498	31.92771084
499	31.86372745
500	32.00000000
501	31.93612774
502	31.87250996
503	31.80914513
504	31.94444444
505	31.88118812
506	32.01581028
507	31.95266272
508	31.88976378
509	31.82711198
510	31.96078431
511	31.89823875
512	31.83593750
513	31.77387914
514	31.71206226
515	31.65048544
516	31.58914729
517	31.72147002
518	31.85328185
519	31.79190751
520	31.73076923
521	31.66986564
522	31.80076628
523	31.93116635
524	32.06106870
525	32.00000000
526	31.93916350
527	32.06831120
528	32.00757576
529	31.94706994
530	32.07547170
531	32.01506591
532	31.95488722
533	31.89493433
534	31.83520599
535	31.96261682
536	31.90298507
537	32.02979516
538	32.15613383
539	32.09647495
540	32.03703704
541	32.16266174
542	32.28782288
543	32.22836096
544	32.16911765
545	32.11009174
546	32.05128205
547	31.99268739
548	32.11678832
549	32.05828780
550	32.00000000
551	31.94192377
552	31.88405797
553	31.82640145
554	31.76895307
555	31.71171171
556	31.65467626
557	31.59784560
558	31.72043011
559	31.66368515
560	31.60714286
561	31.55080214
562	31.49466192
563	31.61634103
564	31.56028369
565	31.50442478
566	31.44876325
567	31.39329806
568	31.51408451
569	31.45869947
570	31.40350877
571	31.52364273
572	31.46853147
573	31.41361257
574	31.53310105
575	31.47826087
576	31.59722222
577	31.71577123
578	31.83391003
579	31.77892919
580	31.72413793
581	31.66953528
582	31.61512027
583	31.56089194
584	31.67808219
585	31.62393162
586	31.56996587
587	31.51618399
588	31.46258503
589	31.40916808
590	31.35593220
591	31.30287648
592	31.25000000
593	31.19730185
594	31.31313131
595	31.26050420
596	31.37583893
597	31.32328308
598	31.27090301
599	31.38564274
600	31.33333333
601	31.44758735
602	31.39534884
603	31.50912106
604	31.45695364
605	31.57024793
606	31.51815182
607	31.63097199
608	31.57894737
609	31.52709360
610	31.47540984
611	31.42389525
612	31.37254902
613	31.32137031
614	31.43322476
615	31.38211382
616	31.49350649
617	31.60453809
618	31.55339806
619	31.50242326
620	31.45161290
621	31.40096618
622	31.51125402
623	31.46067416
624	31.41025641
625	31.36000000
626	31.46964856
627	31.41945774
628	31.36942675
629	31.47853736
630	31.42857143
631	31.37876387
632	31.48734177
633	31.43759874
634	31.38801262
635	31.33858268
636	31.28930818
637	31.24018838
638	31.19122257
639	31.29890454
640	31.40625000
641	31.35725429
642	31.46417445
643	31.41524106
644	31.52173913
645	31.47286822
646	31.42414861
647	31.37557960
648	31.32716049
649	31.27889060
650	31.23076923
651	31.18279570
652	31.28834356
653	31.24042879
654	31.34556575
655	31.29770992
656	31.40243902
657	31.50684932
658	31.61094225
659	31.56297420
660	31.51515152
661	31.46747352
662	31.41993958
663	31.37254902
664	31.32530120
665	31.27819549
666	31.23123123
667	31.33433283
668	31.43712575
669	31.39013453
670	31.34328358
671	31.29657228
672	31.25000000
673	31.20356612
674	31.15727003
675	31.11111111
676	31.06508876
677	31.01920236
678	30.97345133
679	30.92783505
680	31.02941176
681	30.98384728
682	30.93841642
683	31.03953148
684	30.99415205
685	30.94890511
686	30.90379009
687	30.85880640
688	30.81395349
689	30.76923077
690	30.72463768
691	30.82489146
692	30.78034682
693	30.73593074
694	30.69164265
695	30.64748201
696	30.74712644
697	30.70301291
698	30.65902579
699	30.75822604
700	30.71428571
701	30.81312411
702	30.91168091
703	31.00995733
704	30.96590909
705	30.92198582
706	30.87818697
707	30.83451202
708	30.79096045
709	30.74753173
710	30.84507042
711	30.94233474
712	30.89887640
713	30.99579243
714	31.09243697
715	31.04895105
716	31.00558659
717	30.96234310
718	30.91922006
719	30.87621697
720	30.83333333
721	30.79056865
722	30.74792244
723	30.84370678
724	30.80110497
725	30.89655172
726	30.99173554
727	30.94910591
728	30.90659341
729	30.86419753
730	30.82191781
731	30.77975376
732	30.87431694
733	30.96862210
734	30.92643052
735	30.88435374
736	30.84239130
737	30.93622795
738	30.89430894
739	30.98782138
740	31.08108108
741	31.03913630
742	31.13207547
743	31.22476447
744	31.31720430
745	31.27516779
746	31.23324397
747	31.19143240
748	31.14973262
749	31.10814419
750	31.20000000

Final result: 31.2000 ±1.6929
Random chance: 25.0000 ±1.5822