common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 869 tasks in prompt multiple_choice_score: selecting 750 random tasks from 869 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 100.00000000 3 100.00000000 4 75.00000000 5 80.00000000 6 83.33333333 7 85.71428571 8 87.50000000 9 77.77777778 10 80.00000000 11 72.72727273 12 75.00000000 13 76.92307692 14 78.57142857 15 80.00000000 16 75.00000000 17 76.47058824 18 72.22222222 19 73.68421053 20 70.00000000 21 71.42857143 22 72.72727273 23 69.56521739 24 70.83333333 25 68.00000000 26 69.23076923 27 70.37037037 28 67.85714286 29 65.51724138 30 66.66666667 31 64.51612903 32 65.62500000 33 63.63636364 34 61.76470588 35 60.00000000 36 58.33333333 37 59.45945946 38 57.89473684 39 58.97435897 40 60.00000000 41 58.53658537 42 59.52380952 43 58.13953488 44 56.81818182 45 57.77777778 46 58.69565217 47 59.57446809 48 58.33333333 49 59.18367347 50 58.00000000 51 58.82352941 52 59.61538462 53 58.49056604 54 57.40740741 55 56.36363636 56 57.14285714 57 57.89473684 58 58.62068966 59 59.32203390 60 60.00000000 61 60.65573770 62 59.67741935 63 60.31746032 64 60.93750000 65 60.00000000 66 60.60606061 67 61.19402985 68 61.76470588 69 62.31884058 70 61.42857143 71 60.56338028 72 59.72222222 73 60.27397260 74 59.45945946 75 60.00000000 76 59.21052632 77 58.44155844 78 58.97435897 79 58.22784810 80 58.75000000 81 58.02469136 82 58.53658537 83 57.83132530 84 58.33333333 85 58.82352941 86 59.30232558 87 58.62068966 88 59.09090909 89 59.55056180 90 60.00000000 91 60.43956044 92 60.86956522 93 60.21505376 94 59.57446809 95 60.00000000 96 60.41666667 97 59.79381443 98 60.20408163 99 60.60606061 100 60.00000000 101 59.40594059 102 58.82352941 103 58.25242718 104 58.65384615 105 59.04761905 106 58.49056604 107 58.87850467 108 58.33333333 109 57.79816514 110 58.18181818 111 58.55855856 112 58.92857143 113 59.29203540 114 59.64912281 115 60.00000000 116 60.34482759 117 59.82905983 118 60.16949153 119 59.66386555 120 60.00000000 121 59.50413223 122 59.83606557 123 60.16260163 124 59.67741935 125 60.00000000 126 60.31746032 127 60.62992126 128 60.93750000 129 60.46511628 130 60.76923077 131 61.06870229 132 60.60606061 133 60.90225564 134 60.44776119 135 60.00000000 136 60.29411765 137 60.58394161 138 60.86956522 139 61.15107914 140 61.42857143 141 61.70212766 142 61.97183099 143 62.23776224 144 62.50000000 145 62.75862069 146 62.32876712 147 61.90476190 148 62.16216216 149 62.41610738 150 62.66666667 151 62.25165563 152 62.50000000 153 62.74509804 154 62.33766234 155 61.93548387 156 62.17948718 157 62.42038217 158 62.02531646 159 62.26415094 160 61.87500000 161 62.11180124 162 62.34567901 163 62.57668712 164 62.19512195 165 61.81818182 166 62.04819277 167 61.67664671 168 61.90476190 169 61.53846154 170 61.76470588 171 61.40350877 172 61.62790698 173 61.84971098 174 62.06896552 175 61.71428571 176 61.93181818 177 62.14689266 178 61.79775281 179 62.01117318 180 62.22222222 181 61.87845304 182 62.08791209 183 61.74863388 184 61.95652174 185 62.16216216 186 62.36559140 187 62.56684492 188 62.23404255 189 61.90476190 190 62.10526316 191 62.30366492 192 62.50000000 193 62.17616580 194 62.37113402 195 62.56410256 196 62.75510204 197 62.94416244 198 62.62626263 199 62.31155779 200 62.50000000 201 62.68656716 202 62.87128713 203 63.05418719 204 63.23529412 205 62.92682927 206 62.62135922 207 62.31884058 208 62.50000000 209 62.67942584 210 62.85714286 211 63.03317536 212 62.73584906 213 62.91079812 214 63.08411215 215 63.25581395 216 62.96296296 217 62.67281106 218 62.84403670 219 62.55707763 220 62.27272727 221 61.99095023 222 62.16216216 223 61.88340807 224 61.60714286 225 61.33333333 226 61.50442478 227 61.67400881 228 61.84210526 229 62.00873362 230 62.17391304 231 61.90476190 232 61.63793103 233 61.80257511 234 61.96581197 235 61.70212766 236 61.44067797 237 61.60337553 238 61.76470588 239 61.92468619 240 62.08333333 241 62.24066390 242 61.98347107 243 62.13991770 244 61.88524590 245 62.04081633 246 62.19512195 247 62.34817814 248 62.50000000 249 62.24899598 250 62.40000000 251 62.54980080 252 62.69841270 253 62.84584980 254 62.59842520 255 62.74509804 256 62.50000000 257 62.25680934 258 62.40310078 259 62.16216216 260 62.30769231 261 62.45210728 262 62.59541985 263 62.73764259 264 62.87878788 265 62.64150943 266 62.40601504 267 62.54681648 268 62.68656716 269 62.82527881 270 62.96296296 271 62.73062731 272 62.50000000 273 62.27106227 274 62.04379562 275 61.81818182 276 61.95652174 277 62.09386282 278 61.87050360 279 62.00716846 280 62.14285714 281 62.27758007 282 62.41134752 283 62.19081272 284 62.32394366 285 62.45614035 286 62.58741259 287 62.71777003 288 62.84722222 289 62.62975779 290 62.75862069 291 62.88659794 292 63.01369863 293 63.13993174 294 62.92517007 295 63.05084746 296 63.17567568 297 63.29966330 298 63.42281879 299 63.54515050 300 63.33333333 301 63.12292359 302 63.24503311 303 63.36633663 304 63.48684211 305 63.27868852 306 63.07189542 307 62.86644951 308 62.98701299 309 63.10679612 310 63.22580645 311 63.34405145 312 63.14102564 313 63.25878594 314 63.05732484 315 63.17460317 316 63.29113924 317 63.40694006 318 63.20754717 319 63.00940439 320 63.12500000 321 63.23987539 322 63.04347826 323 63.15789474 324 63.27160494 325 63.38461538 326 63.49693252 327 63.30275229 328 63.41463415 329 63.52583587 330 63.33333333 331 63.44410876 332 63.25301205 333 63.36336336 334 63.17365269 335 62.98507463 336 62.79761905 337 62.90801187 338 62.72189349 339 62.83185841 340 62.64705882 341 62.46334311 342 62.28070175 343 62.39067055 344 62.50000000 345 62.31884058 346 62.13872832 347 61.95965418 348 61.78160920 349 61.89111748 350 61.71428571 351 61.53846154 352 61.36363636 353 61.47308782 354 61.29943503 355 61.12676056 356 61.23595506 357 61.34453782 358 61.45251397 359 61.28133705 360 61.38888889 361 61.49584488 362 61.60220994 363 61.43250689 364 61.26373626 365 61.36986301 366 61.47540984 367 61.58038147 368 61.68478261 369 61.78861789 370 61.89189189 371 61.72506739 372 61.82795699 373 61.66219839 374 61.76470588 375 61.60000000 376 61.70212766 377 61.80371353 378 61.90476190 379 61.74142480 380 61.84210526 381 61.94225722 382 61.78010471 383 61.61879896 384 61.71875000 385 61.55844156 386 61.65803109 387 61.49870801 388 61.34020619 389 61.43958869 390 61.28205128 391 61.12531969 392 61.22448980 393 61.32315522 394 61.42131980 395 61.51898734 396 61.36363636 397 61.20906801 398 61.05527638 399 61.15288221 400 61.25000000 401 61.34663342 402 61.44278607 403 61.53846154 404 61.63366337 405 61.72839506 406 61.82266010 407 61.91646192 408 62.00980392 409 61.85819071 410 61.95121951 411 62.04379562 412 61.89320388 413 61.74334140 414 61.83574879 415 61.68674699 416 61.77884615 417 61.87050360 418 61.96172249 419 61.81384248 420 61.90476190 421 61.99524941 422 61.84834123 423 61.93853428 424 61.79245283 425 61.88235294 426 61.97183099 427 62.06088993 428 61.91588785 429 62.00466200 430 62.09302326 431 62.18097448 432 62.26851852 433 62.12471132 434 62.21198157 435 62.29885057 436 62.38532110 437 62.47139588 438 62.55707763 439 62.64236902 440 62.50000000 441 62.58503401 442 62.44343891 443 62.30248307 444 62.16216216 445 62.24719101 446 62.33183857 447 62.41610738 448 62.50000000 449 62.58351893 450 62.44444444 451 62.52771619 452 62.61061947 453 62.47240618 454 62.55506608 455 62.63736264 456 62.71929825 457 62.80087527 458 62.88209607 459 62.96296296 460 63.04347826 461 62.90672451 462 62.98701299 463 62.85097192 464 62.71551724 465 62.79569892 466 62.87553648 467 62.95503212 468 63.03418803 469 62.89978678 470 62.97872340 471 63.05732484 472 63.13559322 473 63.00211416 474 63.08016878 475 63.15789474 476 63.23529412 477 63.31236897 478 63.38912134 479 63.46555324 480 63.54166667 481 63.40956341 482 63.48547718 483 63.56107660 484 63.63636364 485 63.71134021 486 63.78600823 487 63.65503080 488 63.72950820 489 63.59918200 490 63.46938776 491 63.54378819 492 63.61788618 493 63.48884381 494 63.56275304 495 63.43434343 496 63.30645161 497 63.38028169 498 63.45381526 499 63.52705411 500 63.60000000 501 63.47305389 502 63.54581673 503 63.61829026 504 63.49206349 505 63.56435644 506 63.43873518 507 63.31360947 508 63.38582677 509 63.45776031 510 63.52941176 511 63.40508806 512 63.28125000 513 63.15789474 514 63.22957198 515 63.30097087 516 63.17829457 517 63.24951644 518 63.32046332 519 63.39113680 520 63.26923077 521 63.33973129 522 63.40996169 523 63.28871893 524 63.16793893 525 63.04761905 526 62.92775665 527 62.80834915 528 62.68939394 529 62.57088847 530 62.64150943 531 62.71186441 532 62.78195489 533 62.85178236 534 62.73408240 535 62.61682243 536 62.50000000 537 62.56983240 538 62.45353160 539 62.52319109 540 62.40740741 541 62.47689464 542 62.54612546 543 62.61510129 544 62.68382353 545 62.75229358 546 62.63736264 547 62.52285192 548 62.59124088 549 62.47723133 550 62.54545455 551 62.43194192 552 62.31884058 553 62.38698011 554 62.45487365 555 62.52252252 556 62.41007194 557 62.29802513 558 62.18637993 559 62.25402504 560 62.32142857 561 62.38859180 562 62.45551601 563 62.52220249 564 62.41134752 565 62.30088496 566 62.36749117 567 62.43386243 568 62.50000000 569 62.39015817 570 62.28070175 571 62.34676007 572 62.23776224 573 62.30366492 574 62.36933798 575 62.26086957 576 62.32638889 577 62.39168111 578 62.45674740 579 62.52158895 580 62.41379310 581 62.47848537 582 62.54295533 583 62.43567753 584 62.50000000 585 62.56410256 586 62.45733788 587 62.52129472 588 62.58503401 589 62.64855688 590 62.71186441 591 62.77495770 592 62.83783784 593 62.90050590 594 62.96296296 595 63.02521008 596 62.91946309 597 62.81407035 598 62.87625418 599 62.93823038 600 62.83333333 601 62.89517471 602 62.79069767 603 62.68656716 604 62.74834437 605 62.80991736 606 62.87128713 607 62.93245470 608 62.99342105 609 62.88998358 610 62.95081967 611 62.84779051 612 62.74509804 613 62.64274062 614 62.70358306 615 62.76422764 616 62.66233766 617 62.72285251 618 62.62135922 619 62.52019386 620 62.58064516 621 62.64090177 622 62.54019293 623 62.60032103 624 62.66025641 625 62.56000000 626 62.46006390 627 62.51993620 628 62.57961783 629 62.63910970 630 62.69841270 631 62.75752773 632 62.65822785 633 62.55924171 634 62.61829653 635 62.67716535 636 62.73584906 637 62.63736264 638 62.69592476 639 62.75430360 640 62.65625000 641 62.71450858 642 62.61682243 643 62.67496112 644 62.73291925 645 62.63565891 646 62.53869969 647 62.59659969 648 62.50000000 649 62.40369800 650 62.46153846 651 62.51920123 652 62.57668712 653 62.48085758 654 62.38532110 655 62.29007634 656 62.19512195 657 62.10045662 658 62.15805471 659 62.06373293 660 62.12121212 661 62.17851740 662 62.23564955 663 62.29260935 664 62.34939759 665 62.40601504 666 62.46246246 667 62.51874063 668 62.42514970 669 62.33183857 670 62.38805970 671 62.29508197 672 62.35119048 673 62.25854383 674 62.31454006 675 62.22222222 676 62.13017751 677 62.18611521 678 62.24188791 679 62.15022091 680 62.20588235 681 62.11453744 682 62.02346041 683 61.93265007 684 61.84210526 685 61.89781022 686 61.80758017 687 61.86317322 688 61.91860465 689 61.82873730 690 61.88405797 691 61.93921852 692 61.99421965 693 61.90476190 694 61.81556196 695 61.72661871 696 61.63793103 697 61.54949785 698 61.46131805 699 61.37339056 700 61.42857143 701 61.48359486 702 61.53846154 703 61.59317212 704 61.64772727 705 61.56028369 706 61.61473088 707 61.66902405 708 61.72316384 709 61.77715092 710 61.83098592 711 61.88466948 712 61.93820225 713 61.99158485 714 61.90476190 715 61.81818182 716 61.87150838 717 61.92468619 718 61.83844011 719 61.75243394 720 61.80555556 721 61.85852982 722 61.91135734 723 61.96403873 724 61.87845304 725 61.93103448 726 61.98347107 727 61.89821183 728 61.95054945 729 62.00274348 730 62.05479452 731 62.10670315 732 62.15846995 733 62.21009550 734 62.26158038 735 62.17687075 736 62.22826087 737 62.14382632 738 62.19512195 739 62.11096076 740 62.02702703 741 62.07827260 742 62.12938005 743 62.18034993 744 62.09677419 745 62.14765101 746 62.19839142 747 62.11512718 748 62.03208556 749 61.94926569 750 62.00000000 Final result: 62.0000 +/- 1.7736 Random chance: 25.0083 +/- 1.5824