common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 37.50000000 9 33.33333333 10 30.00000000 11 27.27272727 12 25.00000000 13 30.76923077 14 28.57142857 15 33.33333333 16 37.50000000 17 35.29411765 18 33.33333333 19 36.84210526 20 35.00000000 21 33.33333333 22 31.81818182 23 30.43478261 24 29.16666667 25 32.00000000 26 30.76923077 27 29.62962963 28 32.14285714 29 34.48275862 30 36.66666667 31 38.70967742 32 40.62500000 33 39.39393939 34 38.23529412 35 40.00000000 36 38.88888889 37 37.83783784 38 36.84210526 39 35.89743590 40 35.00000000 41 34.14634146 42 33.33333333 43 32.55813953 44 31.81818182 45 31.11111111 46 30.43478261 47 29.78723404 48 31.25000000 49 32.65306122 50 34.00000000 51 33.33333333 52 32.69230769 53 32.07547170 54 31.48148148 55 32.72727273 56 33.92857143 57 35.08771930 58 34.48275862 59 33.89830508 60 33.33333333 61 32.78688525 62 32.25806452 63 31.74603175 64 31.25000000 65 32.30769231 66 33.33333333 67 32.83582090 68 32.35294118 69 31.88405797 70 32.85714286 71 32.39436620 72 33.33333333 73 32.87671233 74 33.78378378 75 34.66666667 76 35.52631579 77 36.36363636 78 37.17948718 79 36.70886076 80 37.50000000 81 38.27160494 82 37.80487805 83 38.55421687 84 39.28571429 85 38.82352941 86 38.37209302 87 39.08045977 88 39.77272727 89 39.32584270 90 38.88888889 91 38.46153846 92 38.04347826 93 37.63440860 94 38.29787234 95 37.89473684 96 38.54166667 97 38.14432990 98 37.75510204 99 37.37373737 100 38.00000000 101 37.62376238 102 37.25490196 103 36.89320388 104 36.53846154 105 37.14285714 106 36.79245283 107 36.44859813 108 36.11111111 109 35.77981651 110 35.45454545 111 35.13513514 112 34.82142857 113 34.51327434 114 35.08771930 115 35.65217391 116 35.34482759 117 35.04273504 118 34.74576271 119 34.45378151 120 34.16666667 121 33.88429752 122 33.60655738 123 33.33333333 124 33.06451613 125 33.60000000 126 33.33333333 127 33.07086614 128 32.81250000 129 32.55813953 130 32.30769231 131 32.82442748 132 33.33333333 133 33.83458647 134 33.58208955 135 33.33333333 136 33.08823529 137 32.84671533 138 32.60869565 139 32.37410072 140 32.14285714 141 32.62411348 142 32.39436620 143 32.86713287 144 33.33333333 145 33.10344828 146 32.87671233 147 32.65306122 148 33.10810811 149 32.88590604 150 32.66666667 151 32.45033113 152 32.23684211 153 32.02614379 154 31.81818182 155 31.61290323 156 31.41025641 157 31.84713376 158 31.64556962 159 31.44654088 160 31.87500000 161 31.67701863 162 31.48148148 163 31.28834356 164 31.70731707 165 32.12121212 166 31.92771084 167 32.33532934 168 32.14285714 169 31.95266272 170 31.76470588 171 31.57894737 172 31.39534884 173 31.21387283 174 31.03448276 175 30.85714286 176 30.68181818 177 30.50847458 178 30.33707865 179 30.72625698 180 30.55555556 181 30.38674033 182 30.21978022 183 30.05464481 184 29.89130435 185 30.27027027 186 30.64516129 187 30.48128342 188 30.31914894 189 30.15873016 190 30.00000000 191 30.36649215 192 30.20833333 193 30.05181347 194 30.41237113 195 30.76923077 196 30.61224490 197 30.45685279 198 30.30303030 199 30.15075377 200 30.00000000 201 30.34825871 202 30.19801980 203 30.04926108 204 29.90196078 205 30.24390244 206 30.09708738 207 29.95169082 208 29.80769231 209 29.66507177 210 29.52380952 211 29.85781991 212 29.71698113 213 29.57746479 214 29.43925234 215 29.30232558 216 29.62962963 217 29.49308756 218 29.35779817 219 29.22374429 220 29.09090909 221 28.95927602 222 29.27927928 223 29.14798206 224 29.01785714 225 29.33333333 226 29.20353982 227 29.51541850 228 29.82456140 229 29.69432314 230 29.56521739 231 29.43722944 232 29.74137931 233 29.61373391 234 29.91452991 235 29.78723404 236 29.66101695 237 29.53586498 238 29.41176471 239 29.28870293 240 29.58333333 241 29.46058091 242 29.75206612 243 30.04115226 244 29.91803279 245 30.20408163 246 30.08130081 247 29.95951417 248 30.24193548 249 30.12048193 250 30.00000000 251 29.88047809 252 29.76190476 253 29.64426877 254 29.52755906 255 29.80392157 256 29.68750000 257 29.96108949 258 29.84496124 259 29.72972973 260 29.61538462 261 29.50191571 262 29.38931298 263 29.27756654 264 29.16666667 265 29.05660377 266 29.32330827 267 29.21348315 268 29.10447761 269 28.99628253 270 28.88888889 271 28.78228782 272 29.04411765 273 28.93772894 274 28.83211679 275 28.72727273 276 28.98550725 277 29.24187726 278 29.49640288 279 29.39068100 280 29.64285714 281 29.53736655 282 29.43262411 283 29.32862191 284 29.22535211 285 29.12280702 286 29.02097902 287 29.26829268 288 29.16666667 289 29.06574394 290 29.31034483 291 29.55326460 292 29.79452055 293 30.03412969 294 30.27210884 295 30.50847458 296 30.40540541 297 30.30303030 298 30.53691275 299 30.76923077 300 31.00000000 301 31.22923588 302 31.45695364 303 31.35313531 304 31.57894737 305 31.80327869 306 31.69934641 307 31.59609121 308 31.49350649 309 31.39158576 310 31.29032258 311 31.51125402 312 31.73076923 313 31.94888179 314 32.16560510 315 32.06349206 316 31.96202532 317 31.86119874 318 31.76100629 319 31.97492163 320 31.87500000 321 31.77570093 322 31.67701863 323 31.57894737 324 31.48148148 325 31.69230769 326 31.59509202 327 31.80428135 328 32.01219512 329 31.91489362 330 31.81818182 331 31.72205438 332 31.92771084 333 31.83183183 334 31.73652695 335 31.94029851 336 31.84523810 337 31.75074184 338 31.65680473 339 31.56342183 340 31.76470588 341 31.67155425 342 31.57894737 343 31.77842566 344 31.68604651 345 31.59420290 346 31.79190751 347 31.98847262 348 31.89655172 349 31.80515759 350 32.00000000 351 31.90883191 352 31.81818182 353 32.01133144 354 31.92090395 355 31.83098592 356 32.02247191 357 31.93277311 358 31.84357542 359 31.75487465 360 31.94444444 361 32.13296399 362 32.32044199 363 32.50688705 364 32.41758242 365 32.32876712 366 32.24043716 367 32.15258856 368 32.06521739 369 31.97831978 370 32.16216216 371 32.07547170 372 31.98924731 373 31.90348525 374 32.08556150 375 32.26666667 376 32.18085106 377 32.09549072 378 32.27513228 379 32.18997361 380 32.10526316 381 32.28346457 382 32.46073298 383 32.63707572 384 32.55208333 385 32.72727273 386 32.64248705 387 32.55813953 388 32.47422680 389 32.64781491 390 32.56410256 391 32.73657289 392 32.90816327 393 32.82442748 394 32.74111675 395 32.65822785 396 32.82828283 397 32.74559194 398 32.66331658 399 32.83208020 400 32.75000000 401 32.91770574 402 33.08457711 403 33.00248139 404 32.92079208 405 32.83950617 406 32.75862069 407 32.67813268 408 32.59803922 409 32.76283619 410 32.68292683 411 32.60340633 412 32.52427184 413 32.68765133 414 32.85024155 415 33.01204819 416 33.17307692 417 33.09352518 418 33.01435407 419 33.17422434 420 33.33333333 421 33.25415677 422 33.17535545 423 33.09692671 424 33.25471698 425 33.41176471 426 33.56807512 427 33.48946136 428 33.41121495 429 33.56643357 430 33.48837209 431 33.64269142 432 33.79629630 433 33.71824480 434 33.87096774 435 33.79310345 436 33.71559633 437 33.63844394 438 33.78995434 439 33.94077449 440 33.86363636 441 33.78684807 442 33.71040724 443 33.63431151 444 33.55855856 445 33.48314607 446 33.40807175 447 33.33333333 448 33.25892857 449 33.18485523 450 33.11111111 451 33.03769401 452 33.18584071 453 33.33333333 454 33.48017621 455 33.40659341 456 33.55263158 457 33.69803063 458 33.62445415 459 33.76906318 460 33.91304348 461 33.83947939 462 33.76623377 463 33.90928726 464 34.05172414 465 33.97849462 466 33.90557940 467 34.04710921 468 33.97435897 469 34.11513859 470 34.25531915 471 34.18259023 472 34.11016949 473 34.24947146 474 34.17721519 475 34.10526316 476 34.24369748 477 34.17190776 478 34.30962343 479 34.44676409 480 34.37500000 481 34.51143451 482 34.43983402 483 34.36853002 484 34.29752066 485 34.22680412 486 34.15637860 487 34.29158111 488 34.22131148 489 34.15132924 490 34.28571429 491 34.41955193 492 34.34959350 493 34.27991886 494 34.41295547 495 34.54545455 496 34.67741935 497 34.60764588 498 34.53815261 499 34.46893788 500 34.60000000 501 34.53093812 502 34.66135458 503 34.59244533 504 34.72222222 505 34.65346535 506 34.78260870 507 34.71400394 508 34.64566929 509 34.57760314 510 34.70588235 511 34.63796477 512 34.57031250 513 34.50292398 514 34.43579767 515 34.36893204 516 34.30232558 517 34.42940039 518 34.55598456 519 34.48940270 520 34.42307692 521 34.35700576 522 34.29118774 523 34.41682600 524 34.54198473 525 34.47619048 526 34.41064639 527 34.53510436 528 34.46969697 529 34.59357278 530 34.71698113 531 34.65160075 532 34.58646617 533 34.70919325 534 34.64419476 535 34.76635514 536 34.88805970 537 35.00931099 538 35.13011152 539 35.06493506 540 35.00000000 541 34.93530499 542 35.05535055 543 34.99079190 544 34.92647059 545 34.86238532 546 34.79853480 547 34.73491773 548 34.85401460 549 34.79052823 550 34.72727273 551 34.84573503 552 34.78260870 553 34.71971067 554 34.65703971 555 34.59459459 556 34.53237410 557 34.47037702 558 34.58781362 559 34.52593918 560 34.46428571 561 34.40285205 562 34.34163701 563 34.45825933 564 34.39716312 565 34.51327434 566 34.45229682 567 34.39153439 568 34.50704225 569 34.44639719 570 34.38596491 571 34.50087566 572 34.44055944 573 34.38045375 574 34.49477352 575 34.43478261 576 34.54861111 577 34.66204506 578 34.60207612 579 34.54231434 580 34.48275862 581 34.42340792 582 34.36426117 583 34.47684391 584 34.58904110 585 34.52991453 586 34.47098976 587 34.41226576 588 34.35374150 589 34.29541596 590 34.23728814 591 34.17935702 592 34.12162162 593 34.06408094 594 34.17508418 595 34.11764706 596 34.22818792 597 34.17085427 598 34.11371237 599 34.22370618 600 34.16666667 601 34.27620632 602 34.21926910 603 34.32835821 604 34.27152318 605 34.21487603 606 34.15841584 607 34.26688633 608 34.21052632 609 34.15435140 610 34.09836066 611 34.04255319 612 33.98692810 613 34.09461664 614 34.20195440 615 34.14634146 616 34.25324675 617 34.35980551 618 34.30420712 619 34.24878837 620 34.19354839 621 34.13848631 622 34.08360129 623 34.02889246 624 33.97435897 625 33.92000000 626 34.02555911 627 33.97129187 628 33.91719745 629 34.02225755 630 34.12698413 631 34.07290016 632 34.17721519 633 34.12322275 634 34.06940063 635 34.01574803 636 33.96226415 637 33.90894819 638 33.85579937 639 33.80281690 640 33.90625000 641 33.85335413 642 33.95638629 643 33.90357698 644 34.00621118 645 33.95348837 646 33.90092879 647 33.84853168 648 33.79629630 649 33.74422188 650 33.69230769 651 33.64055300 652 33.74233129 653 33.69065850 654 33.79204893 655 33.74045802 656 33.84146341 657 33.78995434 658 33.89057751 659 33.83915023 660 33.78787879 661 33.73676248 662 33.68580060 663 33.63499246 664 33.73493976 665 33.68421053 666 33.63363363 667 33.58320840 668 33.68263473 669 33.63228700 670 33.73134328 671 33.68107303 672 33.63095238 673 33.58098068 674 33.53115727 675 33.48148148 676 33.43195266 677 33.38257016 678 33.33333333 679 33.28424153 680 33.38235294 681 33.33333333 682 33.28445748 683 33.38213763 684 33.33333333 685 33.28467153 686 33.23615160 687 33.33333333 688 33.28488372 689 33.23657475 690 33.18840580 691 33.14037627 692 33.09248555 693 33.04473304 694 32.99711816 695 32.94964029 696 33.04597701 697 32.99856528 698 32.95128940 699 33.04721030 700 33.00000000 701 33.09557775 702 33.19088319 703 33.28591750 704 33.23863636 705 33.19148936 706 33.14447592 707 33.09759547 708 33.05084746 709 33.00423131 710 33.09859155 711 33.19268636 712 33.14606742 713 33.23983170 714 33.33333333 715 33.28671329 716 33.24022346 717 33.19386332 718 33.14763231 719 33.10152990 720 33.05555556 721 33.00970874 722 32.96398892 723 33.05670816 724 33.01104972 725 33.10344828 726 33.19559229 727 33.14993122 728 33.10439560 729 33.05898491 730 33.01369863 731 32.96853625 732 33.06010929 733 33.15143247 734 33.24250681 735 33.19727891 736 33.15217391 737 33.24287653 738 33.19783198 739 33.28822733 740 33.37837838 741 33.33333333 742 33.42318059 743 33.51278600 744 33.60215054 745 33.55704698 746 33.51206434 747 33.46720214 748 33.42245989 749 33.37783712 750 33.46666667 Final result: 33.4667 ±1.7242 Random chance: 25.0000 ±1.5822