common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 37.50000000 9 33.33333333 10 30.00000000 11 27.27272727 12 25.00000000 13 30.76923077 14 28.57142857 15 33.33333333 16 37.50000000 17 35.29411765 18 33.33333333 19 36.84210526 20 35.00000000 21 33.33333333 22 31.81818182 23 30.43478261 24 29.16666667 25 32.00000000 26 30.76923077 27 29.62962963 28 32.14285714 29 34.48275862 30 36.66666667 31 38.70967742 32 40.62500000 33 39.39393939 34 38.23529412 35 40.00000000 36 38.88888889 37 37.83783784 38 36.84210526 39 35.89743590 40 35.00000000 41 36.58536585 42 35.71428571 43 34.88372093 44 34.09090909 45 33.33333333 46 32.60869565 47 31.91489362 48 33.33333333 49 34.69387755 50 36.00000000 51 35.29411765 52 34.61538462 53 33.96226415 54 33.33333333 55 32.72727273 56 32.14285714 57 31.57894737 58 31.03448276 59 30.50847458 60 30.00000000 61 31.14754098 62 30.64516129 63 30.15873016 64 29.68750000 65 30.76923077 66 30.30303030 67 29.85074627 68 29.41176471 69 28.98550725 70 30.00000000 71 29.57746479 72 30.55555556 73 30.13698630 74 29.72972973 75 29.33333333 76 30.26315789 77 31.16883117 78 32.05128205 79 31.64556962 80 32.50000000 81 33.33333333 82 32.92682927 83 33.73493976 84 34.52380952 85 34.11764706 86 33.72093023 87 34.48275862 88 35.22727273 89 34.83146067 90 34.44444444 91 34.06593407 92 33.69565217 93 33.33333333 94 32.97872340 95 32.63157895 96 33.33333333 97 32.98969072 98 32.65306122 99 32.32323232 100 33.00000000 101 32.67326733 102 32.35294118 103 32.03883495 104 31.73076923 105 31.42857143 106 31.13207547 107 30.84112150 108 31.48148148 109 31.19266055 110 30.90909091 111 30.63063063 112 30.35714286 113 30.08849558 114 30.70175439 115 30.43478261 116 30.17241379 117 29.91452991 118 29.66101695 119 30.25210084 120 30.00000000 121 29.75206612 122 29.50819672 123 29.26829268 124 29.03225806 125 29.60000000 126 29.36507937 127 29.13385827 128 28.90625000 129 28.68217054 130 28.46153846 131 29.00763359 132 29.54545455 133 29.32330827 134 29.10447761 135 28.88888889 136 28.67647059 137 28.46715328 138 28.26086957 139 28.77697842 140 29.28571429 141 29.78723404 142 30.28169014 143 30.76923077 144 30.55555556 145 30.34482759 146 30.13698630 147 29.93197279 148 30.40540541 149 30.20134228 150 30.00000000 151 29.80132450 152 29.60526316 153 30.06535948 154 29.87012987 155 29.67741935 156 29.48717949 157 29.93630573 158 30.37974684 159 30.18867925 160 30.62500000 161 30.43478261 162 30.24691358 163 30.06134969 164 29.87804878 165 30.30303030 166 30.12048193 167 30.53892216 168 30.35714286 169 30.17751479 170 30.00000000 171 29.82456140 172 29.65116279 173 29.47976879 174 29.31034483 175 29.14285714 176 28.97727273 177 28.81355932 178 28.65168539 179 29.05027933 180 28.88888889 181 28.72928177 182 28.57142857 183 28.96174863 184 28.80434783 185 29.18918919 186 29.03225806 187 28.87700535 188 28.72340426 189 28.57142857 190 28.42105263 191 28.79581152 192 28.64583333 193 28.49740933 194 28.86597938 195 29.23076923 196 29.08163265 197 28.93401015 198 28.78787879 199 28.64321608 200 29.00000000 201 29.35323383 202 29.20792079 203 29.06403941 204 28.92156863 205 29.26829268 206 29.12621359 207 28.98550725 208 28.84615385 209 28.70813397 210 28.57142857 211 28.90995261 212 28.77358491 213 28.63849765 214 28.50467290 215 28.37209302 216 28.70370370 217 28.57142857 218 28.44036697 219 28.31050228 220 28.18181818 221 28.05429864 222 28.37837838 223 28.25112108 224 28.57142857 225 28.88888889 226 28.76106195 227 29.07488987 228 29.38596491 229 29.25764192 230 29.13043478 231 29.00432900 232 29.31034483 233 29.18454936 234 29.48717949 235 29.36170213 236 29.23728814 237 29.11392405 238 28.99159664 239 28.87029289 240 28.75000000 241 29.04564315 242 29.33884298 243 29.62962963 244 29.50819672 245 29.79591837 246 29.67479675 247 29.55465587 248 29.43548387 249 29.31726908 250 29.20000000 251 29.08366534 252 28.96825397 253 28.85375494 254 28.74015748 255 29.01960784 256 29.29687500 257 29.57198444 258 29.45736434 259 29.34362934 260 29.23076923 261 29.11877395 262 29.00763359 263 28.89733840 264 28.78787879 265 29.05660377 266 28.94736842 267 28.83895131 268 28.73134328 269 28.62453532 270 28.51851852 271 28.41328413 272 28.67647059 273 28.57142857 274 28.46715328 275 28.36363636 276 28.62318841 277 28.88086643 278 29.13669065 279 29.03225806 280 29.28571429 281 29.18149466 282 29.07801418 283 28.97526502 284 28.87323944 285 28.77192982 286 28.67132867 287 28.91986063 288 28.81944444 289 28.71972318 290 28.96551724 291 29.20962199 292 29.45205479 293 29.69283276 294 29.93197279 295 29.83050847 296 29.72972973 297 29.62962963 298 29.86577181 299 30.10033445 300 30.33333333 301 30.56478405 302 30.79470199 303 30.69306931 304 30.59210526 305 30.49180328 306 30.39215686 307 30.61889251 308 30.51948052 309 30.42071197 310 30.32258065 311 30.22508039 312 30.44871795 313 30.67092652 314 30.57324841 315 30.47619048 316 30.37974684 317 30.28391167 318 30.18867925 319 30.40752351 320 30.31250000 321 30.21806854 322 30.12422360 323 30.03095975 324 29.93827160 325 30.15384615 326 30.06134969 327 30.27522936 328 30.48780488 329 30.39513678 330 30.30303030 331 30.21148036 332 30.42168675 333 30.33033033 334 30.23952096 335 30.44776119 336 30.35714286 337 30.26706231 338 30.17751479 339 30.08849558 340 30.00000000 341 29.91202346 342 29.82456140 343 30.02915452 344 29.94186047 345 29.85507246 346 30.05780347 347 30.25936599 348 30.17241379 349 30.37249284 350 30.57142857 351 30.48433048 352 30.39772727 353 30.59490085 354 30.50847458 355 30.42253521 356 30.61797753 357 30.53221289 358 30.44692737 359 30.36211699 360 30.55555556 361 30.47091413 362 30.38674033 363 30.57851240 364 30.49450549 365 30.41095890 366 30.60109290 367 30.51771117 368 30.43478261 369 30.35230352 370 30.54054054 371 30.45822102 372 30.37634409 373 30.29490617 374 30.21390374 375 30.40000000 376 30.31914894 377 30.23872679 378 30.42328042 379 30.34300792 380 30.26315789 381 30.18372703 382 30.36649215 383 30.54830287 384 30.46875000 385 30.64935065 386 30.56994819 387 30.49095607 388 30.41237113 389 30.59125964 390 30.51282051 391 30.69053708 392 30.86734694 393 30.78880407 394 30.71065990 395 30.88607595 396 31.06060606 397 31.23425693 398 31.15577889 399 31.32832080 400 31.25000000 401 31.42144638 402 31.59203980 403 31.51364764 404 31.43564356 405 31.35802469 406 31.28078818 407 31.20393120 408 31.12745098 409 31.29584352 410 31.21951220 411 31.14355231 412 31.06796117 413 31.23486683 414 31.40096618 415 31.56626506 416 31.49038462 417 31.41486811 418 31.33971292 419 31.50357995 420 31.66666667 421 31.59144893 422 31.51658768 423 31.44208038 424 31.60377358 425 31.52941176 426 31.45539906 427 31.38173302 428 31.30841121 429 31.46853147 430 31.39534884 431 31.55452436 432 31.71296296 433 31.63972286 434 31.56682028 435 31.49425287 436 31.42201835 437 31.35011442 438 31.50684932 439 31.43507973 440 31.36363636 441 31.29251701 442 31.22171946 443 31.15124153 444 31.30630631 445 31.23595506 446 31.16591928 447 31.09619687 448 31.02678571 449 30.95768374 450 30.88888889 451 30.82039911 452 30.97345133 453 31.12582781 454 31.27753304 455 31.20879121 456 31.35964912 457 31.29102845 458 31.22270742 459 31.37254902 460 31.52173913 461 31.45336226 462 31.60173160 463 31.53347732 464 31.46551724 465 31.39784946 466 31.33047210 467 31.47751606 468 31.41025641 469 31.55650320 470 31.48936170 471 31.42250531 472 31.35593220 473 31.50105708 474 31.43459916 475 31.36842105 476 31.51260504 477 31.44654088 478 31.58995816 479 31.73277662 480 31.66666667 481 31.60083160 482 31.53526971 483 31.46997930 484 31.40495868 485 31.34020619 486 31.27572016 487 31.41683778 488 31.35245902 489 31.28834356 490 31.42857143 491 31.36456212 492 31.30081301 493 31.23732252 494 31.37651822 495 31.51515152 496 31.65322581 497 31.58953722 498 31.52610442 499 31.46292585 500 31.60000000 501 31.53692615 502 31.47410359 503 31.41153082 504 31.54761905 505 31.48514851 506 31.62055336 507 31.55818540 508 31.49606299 509 31.43418468 510 31.56862745 511 31.50684932 512 31.44531250 513 31.38401559 514 31.32295720 515 31.26213592 516 31.20155039 517 31.33462282 518 31.27413127 519 31.21387283 520 31.15384615 521 31.09404990 522 31.03448276 523 31.16634799 524 31.29770992 525 31.23809524 526 31.17870722 527 31.30929791 528 31.25000000 529 31.19092628 530 31.32075472 531 31.26177024 532 31.20300752 533 31.14446529 534 31.27340824 535 31.40186916 536 31.34328358 537 31.47113594 538 31.59851301 539 31.53988868 540 31.48148148 541 31.42329020 542 31.54981550 543 31.49171271 544 31.43382353 545 31.37614679 546 31.31868132 547 31.26142596 548 31.38686131 549 31.32969035 550 31.27272727 551 31.21597096 552 31.15942029 553 31.10307414 554 31.04693141 555 30.99099099 556 31.11510791 557 31.05924596 558 31.18279570 559 31.12701252 560 31.07142857 561 31.01604278 562 30.96085409 563 31.08348135 564 31.02836879 565 30.97345133 566 30.91872792 567 30.86419753 568 30.98591549 569 30.93145870 570 31.05263158 571 31.17338004 572 31.11888112 573 31.06457243 574 31.18466899 575 31.13043478 576 31.25000000 577 31.36915078 578 31.31487889 579 31.43350604 580 31.37931034 581 31.32530120 582 31.27147766 583 31.38936535 584 31.50684932 585 31.45299145 586 31.39931741 587 31.34582624 588 31.29251701 589 31.23938879 590 31.18644068 591 31.13367174 592 31.08108108 593 31.02866779 594 31.14478114 595 31.09243697 596 31.20805369 597 31.15577889 598 31.10367893 599 31.05175292 600 31.00000000 601 31.11480865 602 31.06312292 603 31.17744610 604 31.12582781 605 31.23966942 606 31.18811881 607 31.30148270 608 31.25000000 609 31.19868637 610 31.14754098 611 31.09656301 612 31.04575163 613 31.15823817 614 31.27035831 615 31.21951220 616 31.33116883 617 31.28038898 618 31.22977346 619 31.17932149 620 31.12903226 621 31.07890499 622 31.02893891 623 30.97913323 624 30.92948718 625 30.88000000 626 30.83067093 627 30.78149920 628 30.73248408 629 30.84260731 630 30.79365079 631 30.74484945 632 30.85443038 633 30.80568720 634 30.75709779 635 30.70866142 636 30.66037736 637 30.61224490 638 30.56426332 639 30.67292645 640 30.78125000 641 30.73322933 642 30.84112150 643 30.79315708 644 30.90062112 645 30.85271318 646 30.80495356 647 30.75734158 648 30.70987654 649 30.66255778 650 30.61538462 651 30.56835637 652 30.52147239 653 30.47473201 654 30.58103976 655 30.53435115 656 30.64024390 657 30.74581431 658 30.85106383 659 30.80424886 660 30.75757576 661 30.71104387 662 30.66465257 663 30.61840121 664 30.57228916 665 30.52631579 666 30.48048048 667 30.58470765 668 30.68862275 669 30.64275037 670 30.59701493 671 30.55141580 672 30.50595238 673 30.46062407 674 30.41543027 675 30.37037037 676 30.47337278 677 30.42836041 678 30.53097345 679 30.48600884 680 30.58823529 681 30.54331865 682 30.49853372 683 30.60029283 684 30.55555556 685 30.51094891 686 30.46647230 687 30.56768559 688 30.52325581 689 30.62409289 690 30.57971014 691 30.68017366 692 30.63583815 693 30.59163059 694 30.69164265 695 30.64748201 696 30.74712644 697 30.70301291 698 30.65902579 699 30.75822604 700 30.71428571 701 30.81312411 702 30.91168091 703 31.00995733 704 30.96590909 705 30.92198582 706 30.87818697 707 30.83451202 708 30.79096045 709 30.74753173 710 30.84507042 711 30.94233474 712 30.89887640 713 30.99579243 714 31.09243697 715 31.04895105 716 31.00558659 717 30.96234310 718 30.91922006 719 31.01529903 720 30.97222222 721 30.92926491 722 30.88642659 723 30.98201936 724 30.93922652 725 31.03448276 726 31.12947658 727 31.08665750 728 31.04395604 729 31.00137174 730 30.95890411 731 30.91655267 732 31.01092896 733 31.10504775 734 31.19891008 735 31.15646259 736 31.11413043 737 31.20759837 738 31.16531165 739 31.25845737 740 31.35135135 741 31.30904184 742 31.40161725 743 31.49394347 744 31.58602151 745 31.54362416 746 31.50134048 747 31.45917001 748 31.41711230 749 31.37516689 750 31.46666667 Final result: 31.4667 ±1.6968 Random chance: 25.0000 ±1.5822