common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 817 tasks in prompt multiple_choice_score: selecting 750 random tasks from 817 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 25.00000000 5 20.00000000 6 16.66666667 7 14.28571429 8 12.50000000 9 11.11111111 10 10.00000000 11 9.09090909 12 16.66666667 13 15.38461538 14 14.28571429 15 13.33333333 16 12.50000000 17 11.76470588 18 11.11111111 19 15.78947368 20 15.00000000 21 14.28571429 22 13.63636364 23 13.04347826 24 16.66666667 25 16.00000000 26 19.23076923 27 18.51851852 28 17.85714286 29 20.68965517 30 20.00000000 31 22.58064516 32 21.87500000 33 24.24242424 34 23.52941176 35 22.85714286 36 22.22222222 37 24.32432432 38 23.68421053 39 23.07692308 40 22.50000000 41 21.95121951 42 21.42857143 43 23.25581395 44 22.72727273 45 22.22222222 46 23.91304348 47 23.40425532 48 25.00000000 49 24.48979592 50 26.00000000 51 25.49019608 52 25.00000000 53 26.41509434 54 25.92592593 55 27.27272727 56 26.78571429 57 26.31578947 58 25.86206897 59 25.42372881 60 26.66666667 61 26.22950820 62 27.41935484 63 26.98412698 64 26.56250000 65 26.15384615 66 25.75757576 67 25.37313433 68 25.00000000 69 24.63768116 70 25.71428571 71 26.76056338 72 27.77777778 73 28.76712329 74 28.37837838 75 28.00000000 76 28.94736842 77 28.57142857 78 28.20512821 79 27.84810127 80 27.50000000 81 27.16049383 82 26.82926829 83 26.50602410 84 26.19047619 85 25.88235294 86 25.58139535 87 25.28735632 88 25.00000000 89 25.84269663 90 25.55555556 91 25.27472527 92 25.00000000 93 24.73118280 94 24.46808511 95 25.26315789 96 26.04166667 97 25.77319588 98 25.51020408 99 26.26262626 100 27.00000000 101 27.72277228 102 27.45098039 103 28.15533981 104 27.88461538 105 27.61904762 106 28.30188679 107 28.97196262 108 29.62962963 109 29.35779817 110 30.00000000 111 29.72972973 112 29.46428571 113 29.20353982 114 29.82456140 115 29.56521739 116 29.31034483 117 29.05982906 118 28.81355932 119 28.57142857 120 28.33333333 121 28.92561983 122 28.68852459 123 29.26829268 124 29.03225806 125 29.60000000 126 30.15873016 127 29.92125984 128 29.68750000 129 29.45736434 130 29.23076923 131 29.00763359 132 28.78787879 133 28.57142857 134 29.10447761 135 29.62962963 136 29.41176471 137 29.92700730 138 30.43478261 139 30.21582734 140 30.71428571 141 30.49645390 142 30.28169014 143 30.06993007 144 30.55555556 145 30.34482759 146 30.13698630 147 30.61224490 148 30.40540541 149 30.20134228 150 30.00000000 151 30.46357616 152 30.26315789 153 30.06535948 154 29.87012987 155 29.67741935 156 30.12820513 157 29.93630573 158 30.37974684 159 30.18867925 160 30.00000000 161 29.81366460 162 29.62962963 163 30.06134969 164 29.87804878 165 30.30303030 166 30.72289157 167 30.53892216 168 30.35714286 169 30.17751479 170 30.00000000 171 29.82456140 172 29.65116279 173 30.05780347 174 29.88505747 175 29.71428571 176 30.11363636 177 29.94350282 178 29.77528090 179 29.60893855 180 29.44444444 181 29.28176796 182 29.12087912 183 28.96174863 184 28.80434783 185 28.64864865 186 29.03225806 187 28.87700535 188 28.72340426 189 29.10052910 190 28.94736842 191 28.79581152 192 28.64583333 193 28.49740933 194 28.86597938 195 28.71794872 196 28.57142857 197 28.93401015 198 29.29292929 199 29.14572864 200 29.00000000 201 28.85572139 202 29.20792079 203 29.06403941 204 28.92156863 205 29.26829268 206 29.12621359 207 29.46859903 208 29.32692308 209 29.66507177 210 29.52380952 211 29.38388626 212 29.24528302 213 29.10798122 214 28.97196262 215 28.83720930 216 28.70370370 217 28.57142857 218 28.89908257 219 28.76712329 220 28.63636364 221 28.50678733 222 28.37837838 223 28.25112108 224 28.57142857 225 28.44444444 226 28.31858407 227 28.63436123 228 28.50877193 229 28.38427948 230 28.26086957 231 28.13852814 232 28.01724138 233 28.32618026 234 28.20512821 235 28.08510638 236 28.38983051 237 28.69198312 238 28.99159664 239 28.87029289 240 28.75000000 241 28.63070539 242 28.51239669 243 28.39506173 244 28.27868852 245 28.16326531 246 28.04878049 247 27.93522267 248 28.22580645 249 28.51405622 250 28.40000000 251 28.68525896 252 28.57142857 253 28.45849802 254 28.34645669 255 28.23529412 256 28.12500000 257 28.01556420 258 27.90697674 259 27.79922780 260 27.69230769 261 27.96934866 262 27.86259542 263 27.75665399 264 28.03030303 265 28.30188679 266 28.19548872 267 28.46441948 268 28.35820896 269 28.25278810 270 28.14814815 271 28.04428044 272 27.94117647 273 27.83882784 274 27.73722628 275 27.63636364 276 27.53623188 277 27.43682310 278 27.33812950 279 27.24014337 280 27.50000000 281 27.75800712 282 27.65957447 283 27.56183746 284 27.46478873 285 27.36842105 286 27.62237762 287 27.52613240 288 27.43055556 289 27.33564014 290 27.24137931 291 27.49140893 292 27.73972603 293 27.98634812 294 27.89115646 295 28.13559322 296 28.37837838 297 28.61952862 298 28.52348993 299 28.42809365 300 28.33333333 301 28.23920266 302 28.14569536 303 28.05280528 304 27.96052632 305 27.86885246 306 27.77777778 307 27.68729642 308 27.92207792 309 27.83171521 310 27.74193548 311 27.65273312 312 27.56410256 313 27.47603834 314 27.38853503 315 27.30158730 316 27.21518987 317 27.12933754 318 27.04402516 319 26.95924765 320 26.87500000 321 26.79127726 322 27.01863354 323 26.93498452 324 26.85185185 325 26.76923077 326 26.68711656 327 26.60550459 328 26.82926829 329 26.74772036 330 26.96969697 331 26.88821752 332 27.10843373 333 27.32732733 334 27.24550898 335 27.46268657 336 27.67857143 337 27.59643917 338 27.51479290 339 27.72861357 340 27.64705882 341 27.85923754 342 28.07017544 343 27.98833819 344 28.19767442 345 28.40579710 346 28.61271676 347 28.81844380 348 28.73563218 349 28.93982808 350 28.85714286 351 28.77492877 352 28.97727273 353 29.17847025 354 29.09604520 355 29.29577465 356 29.21348315 357 29.41176471 358 29.60893855 359 29.52646240 360 29.44444444 361 29.36288089 362 29.28176796 363 29.47658402 364 29.39560440 365 29.58904110 366 29.50819672 367 29.70027248 368 29.61956522 369 29.53929539 370 29.45945946 371 29.38005391 372 29.56989247 373 29.49061662 374 29.67914439 375 29.60000000 376 29.52127660 377 29.44297082 378 29.62962963 379 29.55145119 380 29.47368421 381 29.39632546 382 29.31937173 383 29.24281984 384 29.16666667 385 29.09090909 386 29.01554404 387 28.94056848 388 29.12371134 389 29.04884319 390 28.97435897 391 28.90025575 392 28.82653061 393 28.75318066 394 28.93401015 395 28.86075949 396 28.78787879 397 28.96725441 398 28.89447236 399 28.82205514 400 28.75000000 401 28.67830424 402 28.85572139 403 28.78411911 404 28.71287129 405 28.64197531 406 28.57142857 407 28.50122850 408 28.43137255 409 28.36185819 410 28.29268293 411 28.22384428 412 28.39805825 413 28.32929782 414 28.50241546 415 28.43373494 416 28.36538462 417 28.29736211 418 28.22966507 419 28.16229117 420 28.33333333 421 28.50356295 422 28.43601896 423 28.36879433 424 28.30188679 425 28.23529412 426 28.16901408 427 28.10304450 428 28.03738318 429 27.97202797 430 27.90697674 431 27.84222738 432 28.00925926 433 28.17551963 434 28.34101382 435 28.27586207 436 28.21100917 437 28.14645309 438 28.08219178 439 28.01822323 440 27.95454545 441 27.89115646 442 28.05429864 443 27.99097065 444 28.15315315 445 28.08988764 446 28.02690583 447 27.96420582 448 27.90178571 449 28.06236080 450 28.00000000 451 27.93791574 452 27.87610619 453 27.81456954 454 27.75330396 455 27.69230769 456 27.63157895 457 27.57111597 458 27.51091703 459 27.45098039 460 27.39130435 461 27.33188720 462 27.27272727 463 27.21382289 464 27.15517241 465 27.31182796 466 27.25321888 467 27.19486081 468 27.13675214 469 27.07889126 470 27.02127660 471 26.96390658 472 26.90677966 473 26.84989429 474 26.79324895 475 26.94736842 476 26.89075630 477 27.04402516 478 26.98744770 479 26.93110647 480 26.87500000 481 26.81912682 482 26.76348548 483 26.70807453 484 26.85950413 485 27.01030928 486 27.16049383 487 27.10472279 488 27.04918033 489 27.19836401 490 27.34693878 491 27.29124236 492 27.43902439 493 27.38336714 494 27.53036437 495 27.67676768 496 27.62096774 497 27.56539235 498 27.51004016 499 27.45490982 500 27.40000000 501 27.34530938 502 27.49003984 503 27.43538767 504 27.57936508 505 27.52475248 506 27.47035573 507 27.41617357 508 27.36220472 509 27.50491159 510 27.64705882 511 27.59295499 512 27.53906250 513 27.48538012 514 27.43190661 515 27.57281553 516 27.51937984 517 27.65957447 518 27.60617761 519 27.55298651 520 27.50000000 521 27.63915547 522 27.58620690 523 27.72466539 524 27.67175573 525 27.80952381 526 27.75665399 527 27.70398482 528 27.84090909 529 27.78827977 530 27.73584906 531 27.87193974 532 27.81954887 533 27.95497186 534 28.08988764 535 28.03738318 536 27.98507463 537 28.11918063 538 28.06691450 539 28.01484230 540 27.96296296 541 27.91127542 542 27.85977860 543 27.80847145 544 27.75735294 545 27.70642202 546 27.83882784 547 27.78793419 548 27.91970803 549 27.86885246 550 27.81818182 551 27.76769510 552 27.89855072 553 27.84810127 554 27.79783394 555 27.74774775 556 27.87769784 557 27.82764811 558 27.77777778 559 27.72808587 560 27.67857143 561 27.62923351 562 27.58007117 563 27.53108348 564 27.65957447 565 27.78761062 566 27.73851590 567 27.68959436 568 27.81690141 569 27.76801406 570 27.71929825 571 27.67075306 572 27.62237762 573 27.57417103 574 27.70034843 575 27.82608696 576 27.77777778 577 27.72963605 578 27.85467128 579 27.80656304 580 27.93103448 581 28.05507745 582 28.00687285 583 27.95883362 584 28.08219178 585 28.03418803 586 28.15699659 587 28.10902896 588 28.06122449 589 28.01358234 590 28.13559322 591 28.08798646 592 28.04054054 593 27.99325464 594 27.94612795 595 27.89915966 596 27.85234899 597 27.97319933 598 27.92642140 599 27.87979967 600 27.83333333 601 27.95341098 602 28.07308970 603 28.02653400 604 27.98013245 605 27.93388430 606 28.05280528 607 28.00658979 608 27.96052632 609 27.91461412 610 27.86885246 611 27.82324059 612 27.94117647 613 27.89559543 614 27.85016287 615 27.80487805 616 27.92207792 617 28.03889789 618 27.99352751 619 28.10985460 620 28.06451613 621 28.01932367 622 28.13504823 623 28.08988764 624 28.04487179 625 28.00000000 626 27.95527157 627 27.91068581 628 28.02547771 629 28.13990461 630 28.09523810 631 28.20919176 632 28.16455696 633 28.12006319 634 28.23343849 635 28.18897638 636 28.30188679 637 28.41444270 638 28.36990596 639 28.32550861 640 28.28125000 641 28.23712949 642 28.34890966 643 28.30482115 644 28.41614907 645 28.52713178 646 28.48297214 647 28.43894900 648 28.54938272 649 28.50539291 650 28.61538462 651 28.72503840 652 28.83435583 653 28.94333844 654 28.89908257 655 28.85496183 656 28.81097561 657 28.91933029 658 28.87537994 659 28.83156297 660 28.78787879 661 28.74432678 662 28.70090634 663 28.65761689 664 28.61445783 665 28.72180451 666 28.82882883 667 28.78560720 668 28.74251497 669 28.69955157 670 28.65671642 671 28.61400894 672 28.57142857 673 28.52897474 674 28.48664688 675 28.59259259 676 28.69822485 677 28.65583456 678 28.61356932 679 28.57142857 680 28.67647059 681 28.63436123 682 28.59237537 683 28.55051245 684 28.65497076 685 28.61313869 686 28.57142857 687 28.52983988 688 28.63372093 689 28.59216255 690 28.55072464 691 28.65412446 692 28.61271676 693 28.57142857 694 28.67435159 695 28.63309353 696 28.73563218 697 28.69440459 698 28.65329513 699 28.75536481 700 28.85714286 701 28.81597718 702 28.77492877 703 28.73399716 704 28.69318182 705 28.65248227 706 28.61189802 707 28.57142857 708 28.53107345 709 28.49083216 710 28.45070423 711 28.55133615 712 28.65168539 713 28.61150070 714 28.57142857 715 28.67132867 716 28.63128492 717 28.59135286 718 28.55153203 719 28.65090403 720 28.61111111 721 28.57142857 722 28.53185596 723 28.49239281 724 28.45303867 725 28.41379310 726 28.51239669 727 28.47317744 728 28.43406593 729 28.39506173 730 28.49315068 731 28.45417237 732 28.55191257 733 28.51296044 734 28.47411444 735 28.43537415 736 28.39673913 737 28.35820896 738 28.31978320 739 28.41677943 740 28.37837838 741 28.47503374 742 28.43665768 743 28.39838493 744 28.49462366 745 28.45637584 746 28.55227882 747 28.51405622 748 28.47593583 749 28.43791722 750 28.40000000 Final result: 28.4000 ±1.6477 Random chance: 19.8992 ±1.4588