scores/deepseek-r1-distill-qwen-7b-q4_k_m-bartowski.mmlu

common_init_from_params: setting dry_penalty_last_n to ctx_size = 768
common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable)

system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 |
multiple_choice_score: there are 1548 tasks in prompt
multiple_choice_score: selecting 750 random tasks from 1548 tasks available
multiple_choice_score: preparing task data...done
multiple_choice_score : calculating TruthfulQA score over 750 tasks.

task	acc_norm
1	100.00000000
2	50.00000000
3	33.33333333
4	50.00000000
5	40.00000000
6	33.33333333
7	42.85714286
8	37.50000000
9	33.33333333
10	30.00000000
11	27.27272727
12	25.00000000
13	30.76923077
14	28.57142857
15	33.33333333
16	37.50000000
17	35.29411765
18	33.33333333
19	36.84210526
20	35.00000000
21	33.33333333
22	31.81818182
23	30.43478261
24	29.16666667
25	32.00000000
26	30.76923077
27	29.62962963
28	32.14285714
29	34.48275862
30	36.66666667
31	38.70967742
32	40.62500000
33	39.39393939
34	38.23529412
35	40.00000000
36	38.88888889
37	37.83783784
38	36.84210526
39	35.89743590
40	35.00000000
41	34.14634146
42	33.33333333
43	32.55813953
44	31.81818182
45	31.11111111
46	30.43478261
47	29.78723404
48	31.25000000
49	32.65306122
50	34.00000000
51	33.33333333
52	32.69230769
53	32.07547170
54	31.48148148
55	32.72727273
56	33.92857143
57	35.08771930
58	34.48275862
59	33.89830508
60	33.33333333
61	32.78688525
62	32.25806452
63	31.74603175
64	31.25000000
65	32.30769231
66	33.33333333
67	32.83582090
68	32.35294118
69	31.88405797
70	32.85714286
71	32.39436620
72	33.33333333
73	32.87671233
74	33.78378378
75	34.66666667
76	35.52631579
77	36.36363636
78	37.17948718
79	36.70886076
80	37.50000000
81	38.27160494
82	37.80487805
83	38.55421687
84	39.28571429
85	38.82352941
86	38.37209302
87	39.08045977
88	39.77272727
89	39.32584270
90	38.88888889
91	38.46153846
92	38.04347826
93	37.63440860
94	38.29787234
95	37.89473684
96	38.54166667
97	38.14432990
98	37.75510204
99	37.37373737
100	38.00000000
101	37.62376238
102	37.25490196
103	36.89320388
104	36.53846154
105	37.14285714
106	36.79245283
107	36.44859813
108	36.11111111
109	35.77981651
110	35.45454545
111	35.13513514
112	34.82142857
113	34.51327434
114	35.08771930
115	35.65217391
116	35.34482759
117	35.04273504
118	34.74576271
119	34.45378151
120	34.16666667
121	33.88429752
122	33.60655738
123	33.33333333
124	33.06451613
125	33.60000000
126	33.33333333
127	33.07086614
128	32.81250000
129	32.55813953
130	32.30769231
131	32.82442748
132	33.33333333
133	33.83458647
134	33.58208955
135	33.33333333
136	33.08823529
137	32.84671533
138	32.60869565
139	32.37410072
140	32.14285714
141	32.62411348
142	32.39436620
143	32.86713287
144	33.33333333
145	33.10344828
146	32.87671233
147	32.65306122
148	33.10810811
149	32.88590604
150	32.66666667
151	32.45033113
152	32.23684211
153	32.02614379
154	31.81818182
155	31.61290323
156	31.41025641
157	31.84713376
158	31.64556962
159	31.44654088
160	31.87500000
161	31.67701863
162	31.48148148
163	31.28834356
164	31.70731707
165	32.12121212
166	31.92771084
167	32.33532934
168	32.14285714
169	31.95266272
170	31.76470588
171	31.57894737
172	31.39534884
173	31.21387283
174	31.03448276
175	30.85714286
176	30.68181818
177	30.50847458
178	30.33707865
179	30.72625698
180	30.55555556
181	30.38674033
182	30.21978022
183	30.05464481
184	29.89130435
185	30.27027027
186	30.64516129
187	30.48128342
188	30.31914894
189	30.15873016
190	30.00000000
191	30.36649215
192	30.20833333
193	30.05181347
194	30.41237113
195	30.76923077
196	30.61224490
197	30.45685279
198	30.30303030
199	30.15075377
200	30.00000000
201	30.34825871
202	30.19801980
203	30.04926108
204	29.90196078
205	30.24390244
206	30.09708738
207	29.95169082
208	29.80769231
209	29.66507177
210	29.52380952
211	29.85781991
212	29.71698113
213	29.57746479
214	29.43925234
215	29.30232558
216	29.62962963
217	29.49308756
218	29.35779817
219	29.22374429
220	29.09090909
221	28.95927602
222	29.27927928
223	29.14798206
224	29.01785714
225	29.33333333
226	29.20353982
227	29.51541850
228	29.82456140
229	29.69432314
230	29.56521739
231	29.43722944
232	29.74137931
233	29.61373391
234	29.91452991
235	29.78723404
236	29.66101695
237	29.53586498
238	29.41176471
239	29.28870293
240	29.58333333
241	29.46058091
242	29.75206612
243	30.04115226
244	29.91803279
245	30.20408163
246	30.08130081
247	29.95951417
248	30.24193548
249	30.12048193
250	30.00000000
251	29.88047809
252	29.76190476
253	29.64426877
254	29.52755906
255	29.80392157
256	29.68750000
257	29.96108949
258	29.84496124
259	29.72972973
260	29.61538462
261	29.50191571
262	29.38931298
263	29.27756654
264	29.16666667
265	29.05660377
266	29.32330827
267	29.21348315
268	29.10447761
269	28.99628253
270	28.88888889
271	28.78228782
272	29.04411765
273	28.93772894
274	28.83211679
275	28.72727273
276	28.98550725
277	29.24187726
278	29.49640288
279	29.39068100
280	29.64285714
281	29.53736655
282	29.43262411
283	29.32862191
284	29.22535211
285	29.12280702
286	29.02097902
287	29.26829268
288	29.16666667
289	29.06574394
290	29.31034483
291	29.55326460
292	29.79452055
293	30.03412969
294	30.27210884
295	30.50847458
296	30.40540541
297	30.30303030
298	30.53691275
299	30.76923077
300	31.00000000
301	31.22923588
302	31.45695364
303	31.35313531
304	31.57894737
305	31.80327869
306	31.69934641
307	31.59609121
308	31.49350649
309	31.39158576
310	31.29032258
311	31.51125402
312	31.73076923
313	31.94888179
314	32.16560510
315	32.06349206
316	31.96202532
317	31.86119874
318	31.76100629
319	31.97492163
320	31.87500000
321	31.77570093
322	31.67701863
323	31.57894737
324	31.48148148
325	31.69230769
326	31.59509202
327	31.80428135
328	32.01219512
329	31.91489362
330	31.81818182
331	31.72205438
332	31.92771084
333	31.83183183
334	31.73652695
335	31.94029851
336	31.84523810
337	31.75074184
338	31.65680473
339	31.56342183
340	31.76470588
341	31.67155425
342	31.57894737
343	31.77842566
344	31.68604651
345	31.59420290
346	31.79190751
347	31.98847262
348	31.89655172
349	31.80515759
350	32.00000000
351	31.90883191
352	31.81818182
353	32.01133144
354	31.92090395
355	31.83098592
356	32.02247191
357	31.93277311
358	31.84357542
359	31.75487465
360	31.94444444
361	32.13296399
362	32.32044199
363	32.50688705
364	32.41758242
365	32.32876712
366	32.24043716
367	32.15258856
368	32.06521739
369	31.97831978
370	32.16216216
371	32.07547170
372	31.98924731
373	31.90348525
374	32.08556150
375	32.26666667
376	32.18085106
377	32.09549072
378	32.27513228
379	32.18997361
380	32.10526316
381	32.28346457
382	32.46073298
383	32.63707572
384	32.55208333
385	32.72727273
386	32.64248705
387	32.55813953
388	32.47422680
389	32.64781491
390	32.56410256
391	32.73657289
392	32.90816327
393	32.82442748
394	32.74111675
395	32.65822785
396	32.82828283
397	32.74559194
398	32.66331658
399	32.83208020
400	32.75000000
401	32.91770574
402	33.08457711
403	33.00248139
404	32.92079208
405	32.83950617
406	32.75862069
407	32.67813268
408	32.59803922
409	32.76283619
410	32.68292683
411	32.60340633
412	32.52427184
413	32.68765133
414	32.85024155
415	33.01204819
416	33.17307692
417	33.09352518
418	33.01435407
419	33.17422434
420	33.33333333
421	33.25415677
422	33.17535545
423	33.09692671
424	33.25471698
425	33.41176471
426	33.56807512
427	33.48946136
428	33.41121495
429	33.56643357
430	33.48837209
431	33.64269142
432	33.79629630
433	33.71824480
434	33.87096774
435	33.79310345
436	33.71559633
437	33.63844394
438	33.78995434
439	33.94077449
440	33.86363636
441	33.78684807
442	33.71040724
443	33.63431151
444	33.55855856
445	33.48314607
446	33.40807175
447	33.33333333
448	33.25892857
449	33.18485523
450	33.11111111
451	33.03769401
452	33.18584071
453	33.33333333
454	33.48017621
455	33.40659341
456	33.55263158
457	33.69803063
458	33.62445415
459	33.76906318
460	33.91304348
461	33.83947939
462	33.76623377
463	33.90928726
464	34.05172414
465	33.97849462
466	33.90557940
467	34.04710921
468	33.97435897
469	34.11513859
470	34.25531915
471	34.18259023
472	34.11016949
473	34.24947146
474	34.17721519
475	34.10526316
476	34.24369748
477	34.17190776
478	34.30962343
479	34.44676409
480	34.37500000
481	34.51143451
482	34.43983402
483	34.36853002
484	34.29752066
485	34.22680412
486	34.15637860
487	34.29158111
488	34.22131148
489	34.15132924
490	34.28571429
491	34.41955193
492	34.34959350
493	34.27991886
494	34.41295547
495	34.54545455
496	34.67741935
497	34.60764588
498	34.53815261
499	34.46893788
500	34.60000000
501	34.53093812
502	34.66135458
503	34.59244533
504	34.72222222
505	34.65346535
506	34.78260870
507	34.71400394
508	34.64566929
509	34.57760314
510	34.70588235
511	34.63796477
512	34.57031250
513	34.50292398
514	34.43579767
515	34.36893204
516	34.30232558
517	34.42940039
518	34.55598456
519	34.48940270
520	34.42307692
521	34.35700576
522	34.29118774
523	34.41682600
524	34.54198473
525	34.47619048
526	34.41064639
527	34.53510436
528	34.46969697
529	34.59357278
530	34.71698113
531	34.65160075
532	34.58646617
533	34.70919325
534	34.64419476
535	34.76635514
536	34.88805970
537	35.00931099
538	35.13011152
539	35.06493506
540	35.00000000
541	34.93530499
542	35.05535055
543	34.99079190
544	34.92647059
545	34.86238532
546	34.79853480
547	34.73491773
548	34.85401460
549	34.79052823
550	34.72727273
551	34.84573503
552	34.78260870
553	34.71971067
554	34.65703971
555	34.59459459
556	34.53237410
557	34.47037702
558	34.58781362
559	34.52593918
560	34.46428571
561	34.40285205
562	34.34163701
563	34.45825933
564	34.39716312
565	34.51327434
566	34.45229682
567	34.39153439
568	34.50704225
569	34.44639719
570	34.38596491
571	34.50087566
572	34.44055944
573	34.38045375
574	34.49477352
575	34.43478261
576	34.54861111
577	34.66204506
578	34.60207612
579	34.54231434
580	34.48275862
581	34.42340792
582	34.36426117
583	34.47684391
584	34.58904110
585	34.52991453
586	34.47098976
587	34.41226576
588	34.35374150
589	34.29541596
590	34.23728814
591	34.17935702
592	34.12162162
593	34.06408094
594	34.17508418
595	34.11764706
596	34.22818792
597	34.17085427
598	34.11371237
599	34.22370618
600	34.16666667
601	34.27620632
602	34.21926910
603	34.32835821
604	34.27152318
605	34.21487603
606	34.15841584
607	34.26688633
608	34.21052632
609	34.15435140
610	34.09836066
611	34.04255319
612	33.98692810
613	34.09461664
614	34.20195440
615	34.14634146
616	34.25324675
617	34.35980551
618	34.30420712
619	34.24878837
620	34.19354839
621	34.13848631
622	34.08360129
623	34.02889246
624	33.97435897
625	33.92000000
626	34.02555911
627	33.97129187
628	33.91719745
629	34.02225755
630	34.12698413
631	34.07290016
632	34.17721519
633	34.12322275
634	34.06940063
635	34.01574803
636	33.96226415
637	33.90894819
638	33.85579937
639	33.80281690
640	33.90625000
641	33.85335413
642	33.95638629
643	33.90357698
644	34.00621118
645	33.95348837
646	33.90092879
647	33.84853168
648	33.79629630
649	33.74422188
650	33.69230769
651	33.64055300
652	33.74233129
653	33.69065850
654	33.79204893
655	33.74045802
656	33.84146341
657	33.78995434
658	33.89057751
659	33.83915023
660	33.78787879
661	33.73676248
662	33.68580060
663	33.63499246
664	33.73493976
665	33.68421053
666	33.63363363
667	33.58320840
668	33.68263473
669	33.63228700
670	33.73134328
671	33.68107303
672	33.63095238
673	33.58098068
674	33.53115727
675	33.48148148
676	33.43195266
677	33.38257016
678	33.33333333
679	33.28424153
680	33.38235294
681	33.33333333
682	33.28445748
683	33.38213763
684	33.33333333
685	33.28467153
686	33.23615160
687	33.33333333
688	33.28488372
689	33.23657475
690	33.18840580
691	33.14037627
692	33.09248555
693	33.04473304
694	32.99711816
695	32.94964029
696	33.04597701
697	32.99856528
698	32.95128940
699	33.04721030
700	33.00000000
701	33.09557775
702	33.19088319
703	33.28591750
704	33.23863636
705	33.19148936
706	33.14447592
707	33.09759547
708	33.05084746
709	33.00423131
710	33.09859155
711	33.19268636
712	33.14606742
713	33.23983170
714	33.33333333
715	33.28671329
716	33.24022346
717	33.19386332
718	33.14763231
719	33.10152990
720	33.05555556
721	33.00970874
722	32.96398892
723	33.05670816
724	33.01104972
725	33.10344828
726	33.19559229
727	33.14993122
728	33.10439560
729	33.05898491
730	33.01369863
731	32.96853625
732	33.06010929
733	33.15143247
734	33.24250681
735	33.19727891
736	33.15217391
737	33.24287653
738	33.19783198
739	33.28822733
740	33.37837838
741	33.33333333
742	33.42318059
743	33.51278600
744	33.60215054
745	33.55704698
746	33.51206434
747	33.46720214
748	33.42245989
749	33.37783712
750	33.46666667

Final result: 33.4667 ±1.7242
Random chance: 25.0000 ±1.5822