common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 50.00000000 9 44.44444444 10 40.00000000 11 36.36363636 12 33.33333333 13 38.46153846 14 35.71428571 15 33.33333333 16 37.50000000 17 35.29411765 18 33.33333333 19 36.84210526 20 35.00000000 21 33.33333333 22 31.81818182 23 30.43478261 24 29.16666667 25 32.00000000 26 30.76923077 27 29.62962963 28 32.14285714 29 34.48275862 30 36.66666667 31 38.70967742 32 40.62500000 33 39.39393939 34 38.23529412 35 40.00000000 36 38.88888889 37 37.83783784 38 39.47368421 39 38.46153846 40 37.50000000 41 36.58536585 42 35.71428571 43 34.88372093 44 34.09090909 45 33.33333333 46 32.60869565 47 31.91489362 48 33.33333333 49 34.69387755 50 36.00000000 51 35.29411765 52 34.61538462 53 33.96226415 54 33.33333333 55 34.54545455 56 35.71428571 57 35.08771930 58 34.48275862 59 33.89830508 60 33.33333333 61 32.78688525 62 32.25806452 63 31.74603175 64 31.25000000 65 30.76923077 66 31.81818182 67 31.34328358 68 30.88235294 69 31.88405797 70 32.85714286 71 32.39436620 72 33.33333333 73 32.87671233 74 32.43243243 75 33.33333333 76 34.21052632 77 35.06493506 78 35.89743590 79 35.44303797 80 36.25000000 81 35.80246914 82 35.36585366 83 36.14457831 84 36.90476190 85 36.47058824 86 36.04651163 87 36.78160920 88 37.50000000 89 37.07865169 90 36.66666667 91 36.26373626 92 35.86956522 93 35.48387097 94 35.10638298 95 34.73684211 96 35.41666667 97 35.05154639 98 34.69387755 99 34.34343434 100 34.00000000 101 33.66336634 102 33.33333333 103 33.00970874 104 32.69230769 105 32.38095238 106 32.07547170 107 31.77570093 108 32.40740741 109 32.11009174 110 31.81818182 111 31.53153153 112 31.25000000 113 31.85840708 114 32.45614035 115 33.04347826 116 32.75862069 117 32.47863248 118 32.20338983 119 31.93277311 120 31.66666667 121 31.40495868 122 31.14754098 123 30.89430894 124 30.64516129 125 31.20000000 126 30.95238095 127 30.70866142 128 30.46875000 129 30.23255814 130 30.00000000 131 30.53435115 132 30.30303030 133 30.07518797 134 29.85074627 135 29.62962963 136 30.14705882 137 29.92700730 138 29.71014493 139 29.49640288 140 29.28571429 141 29.07801418 142 29.57746479 143 30.06993007 144 30.55555556 145 31.03448276 146 30.82191781 147 30.61224490 148 31.08108108 149 30.87248322 150 30.66666667 151 30.46357616 152 30.92105263 153 30.71895425 154 31.16883117 155 30.96774194 156 30.76923077 157 31.21019108 158 31.64556962 159 31.44654088 160 31.87500000 161 31.67701863 162 31.48148148 163 31.28834356 164 31.09756098 165 31.51515152 166 31.32530120 167 31.73652695 168 31.54761905 169 31.36094675 170 31.17647059 171 30.99415205 172 30.81395349 173 30.63583815 174 30.45977011 175 30.28571429 176 30.68181818 177 30.50847458 178 30.33707865 179 30.72625698 180 30.55555556 181 30.38674033 182 30.21978022 183 30.60109290 184 30.43478261 185 30.81081081 186 31.18279570 187 31.01604278 188 30.85106383 189 30.68783069 190 30.52631579 191 30.36649215 192 30.20833333 193 30.05181347 194 30.41237113 195 30.76923077 196 31.12244898 197 30.96446701 198 30.80808081 199 30.65326633 200 30.50000000 201 30.84577114 202 30.69306931 203 31.03448276 204 30.88235294 205 31.21951220 206 31.06796117 207 30.91787440 208 30.76923077 209 30.62200957 210 30.47619048 211 30.80568720 212 30.66037736 213 30.51643192 214 30.37383178 215 30.23255814 216 30.09259259 217 29.95391705 218 29.81651376 219 29.68036530 220 29.54545455 221 29.41176471 222 29.72972973 223 29.59641256 224 29.91071429 225 30.22222222 226 30.08849558 227 30.39647577 228 30.70175439 229 30.56768559 230 30.43478261 231 30.30303030 232 30.17241379 233 30.04291845 234 30.34188034 235 30.21276596 236 30.08474576 237 29.95780591 238 29.83193277 239 29.70711297 240 29.58333333 241 29.46058091 242 29.75206612 243 30.04115226 244 29.91803279 245 30.20408163 246 30.08130081 247 29.95951417 248 29.83870968 249 29.71887550 250 29.60000000 251 29.48207171 252 29.36507937 253 29.64426877 254 29.52755906 255 29.80392157 256 29.68750000 257 29.96108949 258 29.84496124 259 30.11583012 260 30.00000000 261 29.88505747 262 29.77099237 263 29.65779468 264 29.92424242 265 30.18867925 266 30.07518797 267 29.96254682 268 29.85074627 269 29.73977695 270 29.62962963 271 29.52029520 272 29.77941176 273 29.67032967 274 29.56204380 275 29.45454545 276 29.71014493 277 29.96389892 278 30.21582734 279 30.10752688 280 30.00000000 281 29.89323843 282 29.78723404 283 29.68197880 284 29.57746479 285 29.47368421 286 29.37062937 287 29.61672474 288 29.51388889 289 29.41176471 290 29.65517241 291 29.89690722 292 30.13698630 293 30.37542662 294 30.61224490 295 30.50847458 296 30.40540541 297 30.30303030 298 30.53691275 299 30.43478261 300 30.66666667 301 30.56478405 302 30.79470199 303 30.69306931 304 30.92105263 305 31.14754098 306 31.04575163 307 31.27035831 308 31.16883117 309 31.06796117 310 30.96774194 311 30.86816720 312 31.08974359 313 31.30990415 314 31.52866242 315 31.42857143 316 31.32911392 317 31.23028391 318 31.13207547 319 31.34796238 320 31.25000000 321 31.15264798 322 31.05590062 323 30.95975232 324 30.86419753 325 31.07692308 326 30.98159509 327 31.19266055 328 31.09756098 329 31.00303951 330 30.90909091 331 30.81570997 332 31.02409639 333 30.93093093 334 30.83832335 335 31.04477612 336 30.95238095 337 30.86053412 338 30.76923077 339 30.67846608 340 30.58823529 341 30.49853372 342 30.40935673 343 30.32069971 344 30.23255814 345 30.14492754 346 30.34682081 347 30.54755043 348 30.45977011 349 30.65902579 350 30.85714286 351 30.76923077 352 30.68181818 353 30.87818697 354 30.79096045 355 30.70422535 356 30.89887640 357 30.81232493 358 30.72625698 359 30.91922006 360 31.11111111 361 31.30193906 362 31.21546961 363 31.40495868 364 31.31868132 365 31.23287671 366 31.14754098 367 31.06267030 368 30.97826087 369 30.89430894 370 31.08108108 371 30.99730458 372 30.91397849 373 30.83109920 374 31.01604278 375 30.93333333 376 30.85106383 377 30.76923077 378 30.68783069 379 30.60686016 380 30.52631579 381 30.70866142 382 30.62827225 383 30.80939948 384 30.72916667 385 30.64935065 386 30.56994819 387 30.49095607 388 30.41237113 389 30.59125964 390 30.51282051 391 30.69053708 392 30.86734694 393 30.78880407 394 30.71065990 395 30.63291139 396 30.80808081 397 30.73047859 398 30.65326633 399 30.82706767 400 30.75000000 401 30.92269327 402 30.84577114 403 30.76923077 404 30.69306931 405 30.61728395 406 30.78817734 407 30.71253071 408 30.63725490 409 30.80684597 410 30.73170732 411 30.65693431 412 30.58252427 413 30.75060533 414 30.91787440 415 31.08433735 416 31.00961538 417 30.93525180 418 30.86124402 419 31.02625298 420 31.19047619 421 31.11638955 422 31.04265403 423 30.96926714 424 31.13207547 425 31.29411765 426 31.22065728 427 31.14754098 428 31.07476636 429 31.23543124 430 31.16279070 431 31.32250580 432 31.48148148 433 31.40877598 434 31.56682028 435 31.49425287 436 31.42201835 437 31.35011442 438 31.50684932 439 31.66287016 440 31.59090909 441 31.51927438 442 31.44796380 443 31.37697517 444 31.30630631 445 31.23595506 446 31.16591928 447 31.09619687 448 31.02678571 449 30.95768374 450 30.88888889 451 30.82039911 452 30.97345133 453 31.12582781 454 31.05726872 455 30.98901099 456 31.14035088 457 31.07221007 458 31.00436681 459 31.15468410 460 31.30434783 461 31.23644252 462 31.16883117 463 31.31749460 464 31.25000000 465 31.18279570 466 31.11587983 467 31.26338330 468 31.19658120 469 31.34328358 470 31.27659574 471 31.21019108 472 31.14406780 473 31.28964059 474 31.22362869 475 31.15789474 476 31.30252101 477 31.23689727 478 31.38075314 479 31.52400835 480 31.45833333 481 31.39293139 482 31.32780083 483 31.26293996 484 31.19834711 485 31.13402062 486 31.06995885 487 31.21149897 488 31.14754098 489 31.08384458 490 31.22448980 491 31.36456212 492 31.30081301 493 31.23732252 494 31.37651822 495 31.51515152 496 31.65322581 497 31.58953722 498 31.52610442 499 31.46292585 500 31.60000000 501 31.53692615 502 31.47410359 503 31.61033797 504 31.54761905 505 31.48514851 506 31.62055336 507 31.55818540 508 31.49606299 509 31.43418468 510 31.56862745 511 31.50684932 512 31.44531250 513 31.38401559 514 31.32295720 515 31.26213592 516 31.20155039 517 31.33462282 518 31.46718147 519 31.40655106 520 31.34615385 521 31.28598848 522 31.41762452 523 31.54875717 524 31.67938931 525 31.61904762 526 31.55893536 527 31.68880455 528 31.62878788 529 31.56899811 530 31.69811321 531 31.63841808 532 31.57894737 533 31.51969981 534 31.46067416 535 31.40186916 536 31.34328358 537 31.47113594 538 31.59851301 539 31.53988868 540 31.48148148 541 31.60813309 542 31.73431734 543 31.86003683 544 31.80147059 545 31.74311927 546 31.68498168 547 31.62705667 548 31.75182482 549 31.69398907 550 31.63636364 551 31.57894737 552 31.70289855 553 31.64556962 554 31.58844765 555 31.53153153 556 31.47482014 557 31.41831239 558 31.54121864 559 31.48479428 560 31.42857143 561 31.37254902 562 31.31672598 563 31.43872114 564 31.38297872 565 31.32743363 566 31.27208481 567 31.21693122 568 31.33802817 569 31.28295255 570 31.22807018 571 31.34851138 572 31.29370629 573 31.23909250 574 31.35888502 575 31.30434783 576 31.42361111 577 31.54246101 578 31.48788927 579 31.43350604 580 31.37931034 581 31.32530120 582 31.27147766 583 31.21783877 584 31.33561644 585 31.28205128 586 31.22866894 587 31.17546848 588 31.12244898 589 31.06960951 590 31.01694915 591 30.96446701 592 30.91216216 593 30.86003373 594 30.97643098 595 30.92436975 596 31.04026846 597 30.98827471 598 30.93645485 599 31.05175292 600 31.00000000 601 31.11480865 602 31.06312292 603 31.17744610 604 31.12582781 605 31.23966942 606 31.18811881 607 31.30148270 608 31.25000000 609 31.19868637 610 31.14754098 611 31.09656301 612 31.04575163 613 30.99510604 614 31.10749186 615 31.05691057 616 31.16883117 617 31.28038898 618 31.22977346 619 31.17932149 620 31.12903226 621 31.07890499 622 31.18971061 623 31.13964687 624 31.08974359 625 31.04000000 626 31.15015974 627 31.10047847 628 31.05095541 629 31.16057234 630 31.11111111 631 31.06180666 632 31.17088608 633 31.12164297 634 31.07255521 635 31.02362205 636 30.97484277 637 30.92621664 638 30.87774295 639 30.82942097 640 30.93750000 641 30.88923557 642 30.99688474 643 30.94867807 644 31.05590062 645 31.00775194 646 30.95975232 647 30.91190108 648 30.86419753 649 30.81664099 650 30.76923077 651 30.72196621 652 30.67484663 653 30.62787136 654 30.73394495 655 30.68702290 656 30.79268293 657 30.74581431 658 30.85106383 659 30.80424886 660 30.75757576 661 30.71104387 662 30.66465257 663 30.61840121 664 30.57228916 665 30.52631579 666 30.48048048 667 30.58470765 668 30.68862275 669 30.64275037 670 30.74626866 671 30.70044709 672 30.65476190 673 30.60921248 674 30.56379822 675 30.51851852 676 30.47337278 677 30.42836041 678 30.38348083 679 30.33873343 680 30.44117647 681 30.54331865 682 30.49853372 683 30.60029283 684 30.55555556 685 30.51094891 686 30.46647230 687 30.42212518 688 30.37790698 689 30.33381713 690 30.28985507 691 30.39073806 692 30.34682081 693 30.30303030 694 30.40345821 695 30.35971223 696 30.45977011 697 30.41606887 698 30.37249284 699 30.47210300 700 30.42857143 701 30.52781740 702 30.62678063 703 30.72546230 704 30.68181818 705 30.63829787 706 30.59490085 707 30.55162659 708 30.50847458 709 30.46544429 710 30.56338028 711 30.66104079 712 30.61797753 713 30.71528752 714 30.81232493 715 30.76923077 716 30.72625698 717 30.68340307 718 30.64066852 719 30.59805285 720 30.55555556 721 30.51317614 722 30.47091413 723 30.56708160 724 30.52486188 725 30.62068966 726 30.71625344 727 30.67400275 728 30.63186813 729 30.72702332 730 30.68493151 731 30.64295486 732 30.73770492 733 30.83219645 734 30.79019074 735 30.74829932 736 30.70652174 737 30.80054274 738 30.75880759 739 30.85250338 740 30.94594595 741 30.90418354 742 30.99730458 743 31.09017497 744 31.18279570 745 31.14093960 746 31.09919571 747 31.05756359 748 31.01604278 749 31.10814419 750 31.20000000 Final result: 31.2000 ±1.6929 Random chance: 25.0000 ±1.5822