common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 50.00000000 9 44.44444444 10 50.00000000 11 45.45454545 12 41.66666667 13 38.46153846 14 35.71428571 15 40.00000000 16 43.75000000 17 47.05882353 18 44.44444444 19 47.36842105 20 50.00000000 21 52.38095238 22 50.00000000 23 47.82608696 24 50.00000000 25 52.00000000 26 50.00000000 27 51.85185185 28 50.00000000 29 51.72413793 30 53.33333333 31 54.83870968 32 56.25000000 33 54.54545455 34 55.88235294 35 57.14285714 36 55.55555556 37 54.05405405 38 55.26315789 39 53.84615385 40 52.50000000 41 51.21951220 42 50.00000000 43 51.16279070 44 50.00000000 45 51.11111111 46 50.00000000 47 48.93617021 48 50.00000000 49 51.02040816 50 50.00000000 51 49.01960784 52 48.07692308 53 47.16981132 54 46.29629630 55 47.27272727 56 48.21428571 57 47.36842105 58 46.55172414 59 47.45762712 60 46.66666667 61 47.54098361 62 46.77419355 63 46.03174603 64 45.31250000 65 44.61538462 66 45.45454545 67 44.77611940 68 44.11764706 69 43.47826087 70 44.28571429 71 45.07042254 72 44.44444444 73 43.83561644 74 44.59459459 75 45.33333333 76 46.05263158 77 46.75324675 78 47.43589744 79 46.83544304 80 47.50000000 81 48.14814815 82 47.56097561 83 48.19277108 84 48.80952381 85 48.23529412 86 47.67441860 87 48.27586207 88 48.86363636 89 48.31460674 90 47.77777778 91 47.25274725 92 46.73913043 93 46.23655914 94 46.80851064 95 46.31578947 96 46.87500000 97 46.39175258 98 45.91836735 99 46.46464646 100 46.00000000 101 46.53465347 102 46.07843137 103 45.63106796 104 46.15384615 105 46.66666667 106 47.16981132 107 46.72897196 108 47.22222222 109 47.70642202 110 48.18181818 111 48.64864865 112 48.21428571 113 47.78761062 114 48.24561404 115 48.69565217 116 48.27586207 117 47.86324786 118 48.30508475 119 47.89915966 120 47.50000000 121 47.93388430 122 47.54098361 123 47.15447154 124 47.58064516 125 48.00000000 126 47.61904762 127 47.24409449 128 46.87500000 129 46.51162791 130 46.15384615 131 45.80152672 132 46.21212121 133 45.86466165 134 45.52238806 135 45.18518519 136 44.85294118 137 45.25547445 138 44.92753623 139 45.32374101 140 45.00000000 141 45.39007092 142 45.77464789 143 46.15384615 144 46.52777778 145 46.89655172 146 47.26027397 147 46.93877551 148 47.29729730 149 46.97986577 150 46.66666667 151 46.35761589 152 46.71052632 153 46.40522876 154 46.10389610 155 45.80645161 156 45.51282051 157 45.85987261 158 46.20253165 159 45.91194969 160 46.25000000 161 45.96273292 162 45.67901235 163 45.39877301 164 45.12195122 165 45.45454545 166 45.18072289 167 45.50898204 168 45.23809524 169 45.56213018 170 45.29411765 171 45.61403509 172 45.34883721 173 45.08670520 174 44.82758621 175 44.57142857 176 44.31818182 177 44.06779661 178 43.82022472 179 44.13407821 180 44.44444444 181 44.75138122 182 44.50549451 183 44.80874317 184 44.56521739 185 44.86486486 186 45.16129032 187 44.91978610 188 45.21276596 189 45.50264550 190 45.26315789 191 45.54973822 192 45.31250000 193 45.07772021 194 45.36082474 195 45.64102564 196 45.91836735 197 45.68527919 198 45.45454545 199 45.22613065 200 45.00000000 201 44.77611940 202 44.55445545 203 44.33497537 204 44.11764706 205 44.39024390 206 44.17475728 207 44.44444444 208 44.23076923 209 44.01913876 210 43.80952381 211 44.07582938 212 43.86792453 213 43.66197183 214 43.45794393 215 43.72093023 216 43.98148148 217 43.77880184 218 44.03669725 219 44.29223744 220 44.54545455 221 44.34389140 222 44.59459459 223 44.84304933 224 44.64285714 225 44.88888889 226 44.69026549 227 44.49339207 228 44.73684211 229 44.54148472 230 44.34782609 231 44.15584416 232 44.39655172 233 44.20600858 234 44.44444444 235 44.25531915 236 44.06779661 237 43.88185654 238 43.69747899 239 43.51464435 240 43.75000000 241 43.56846473 242 43.80165289 243 44.03292181 244 43.85245902 245 44.08163265 246 43.90243902 247 43.72469636 248 43.95161290 249 43.77510040 250 43.60000000 251 43.42629482 252 43.25396825 253 43.08300395 254 42.91338583 255 42.74509804 256 42.96875000 257 43.19066148 258 43.02325581 259 42.85714286 260 43.07692308 261 42.91187739 262 42.74809160 263 42.58555133 264 42.42424242 265 42.64150943 266 42.85714286 267 42.69662921 268 42.53731343 269 42.75092937 270 42.59259259 271 42.43542435 272 42.64705882 273 42.49084249 274 42.33576642 275 42.18181818 276 42.39130435 277 42.59927798 278 42.44604317 279 42.29390681 280 42.50000000 281 42.34875445 282 42.55319149 283 42.40282686 284 42.25352113 285 42.10526316 286 41.95804196 287 42.16027875 288 42.01388889 289 41.86851211 290 42.06896552 291 42.26804124 292 42.46575342 293 42.32081911 294 42.17687075 295 42.03389831 296 41.89189189 297 41.75084175 298 41.94630872 299 41.80602007 300 42.00000000 301 41.86046512 302 41.72185430 303 41.58415842 304 41.44736842 305 41.63934426 306 41.50326797 307 41.69381107 308 41.55844156 309 41.42394822 310 41.29032258 311 41.15755627 312 41.34615385 313 41.21405751 314 41.08280255 315 40.95238095 316 40.82278481 317 41.00946372 318 41.19496855 319 41.37931034 320 41.25000000 321 41.12149533 322 40.99378882 323 40.86687307 324 40.74074074 325 40.92307692 326 40.79754601 327 40.97859327 328 40.85365854 329 40.72948328 330 40.60606061 331 40.48338369 332 40.66265060 333 40.84084084 334 40.71856287 335 40.89552239 336 41.07142857 337 40.94955490 338 40.82840237 339 40.70796460 340 40.88235294 341 41.05571848 342 41.22807018 343 41.10787172 344 40.98837209 345 40.86956522 346 41.04046243 347 41.21037464 348 41.09195402 349 40.97421203 350 41.14285714 351 41.02564103 352 40.90909091 353 41.07648725 354 40.96045198 355 40.84507042 356 41.01123596 357 41.17647059 358 41.06145251 359 40.94707521 360 41.11111111 361 41.27423823 362 41.43646409 363 41.59779614 364 41.48351648 365 41.36986301 366 41.53005464 367 41.41689373 368 41.30434783 369 41.19241192 370 41.35135135 371 41.23989218 372 41.12903226 373 41.01876676 374 41.17647059 375 41.33333333 376 41.22340426 377 41.11405836 378 41.26984127 379 41.16094987 380 41.05263158 381 41.20734908 382 41.09947644 383 41.25326371 384 41.14583333 385 41.29870130 386 41.19170984 387 41.08527132 388 40.97938144 389 41.13110540 390 41.28205128 391 41.43222506 392 41.58163265 393 41.47582697 394 41.37055838 395 41.51898734 396 41.66666667 397 41.56171285 398 41.45728643 399 41.60401003 400 41.50000000 401 41.64588529 402 41.54228856 403 41.43920596 404 41.33663366 405 41.23456790 406 41.13300493 407 41.03194103 408 40.93137255 409 41.07579462 410 40.97560976 411 40.87591241 412 40.77669903 413 40.67796610 414 40.57971014 415 40.48192771 416 40.62500000 417 40.52757794 418 40.43062201 419 40.57279236 420 40.47619048 421 40.61757720 422 40.52132701 423 40.42553191 424 40.56603774 425 40.47058824 426 40.37558685 427 40.51522248 428 40.42056075 429 40.55944056 430 40.46511628 431 40.60324826 432 40.50925926 433 40.41570439 434 40.32258065 435 40.22988506 436 40.13761468 437 40.04576659 438 40.18264840 439 40.31890661 440 40.22727273 441 40.36281179 442 40.27149321 443 40.18058691 444 40.09009009 445 40.00000000 446 39.91031390 447 39.82102908 448 39.73214286 449 39.64365256 450 39.77777778 451 39.91130820 452 39.82300885 453 39.95584989 454 39.86784141 455 39.78021978 456 39.91228070 457 39.82494530 458 39.73799127 459 39.86928105 460 40.00000000 461 39.91323210 462 40.04329004 463 39.95680346 464 40.08620690 465 40.00000000 466 39.91416309 467 40.04282655 468 40.17094017 469 40.08528785 470 40.00000000 471 39.91507431 472 39.83050847 473 39.95771670 474 39.87341772 475 39.78947368 476 39.91596639 477 39.83228512 478 39.95815900 479 40.08350731 480 40.20833333 481 40.12474012 482 40.04149378 483 39.95859213 484 39.87603306 485 39.79381443 486 39.71193416 487 39.83572895 488 39.75409836 489 39.67280164 490 39.79591837 491 39.71486762 492 39.63414634 493 39.55375254 494 39.67611336 495 39.79797980 496 39.91935484 497 39.83903421 498 39.75903614 499 39.67935872 500 39.80000000 501 39.72055888 502 39.64143426 503 39.76143141 504 39.68253968 505 39.60396040 506 39.72332016 507 39.64497041 508 39.56692913 509 39.48919450 510 39.60784314 511 39.53033268 512 39.45312500 513 39.37621832 514 39.29961089 515 39.41747573 516 39.53488372 517 39.65183752 518 39.76833977 519 39.69171484 520 39.61538462 521 39.53934741 522 39.65517241 523 39.57934990 524 39.69465649 525 39.61904762 526 39.54372624 527 39.65844402 528 39.77272727 529 39.69754253 530 39.81132075 531 39.73634652 532 39.66165414 533 39.58724203 534 39.51310861 535 39.62616822 536 39.55223881 537 39.66480447 538 39.77695167 539 39.70315399 540 39.62962963 541 39.55637708 542 39.66789668 543 39.59484346 544 39.70588235 545 39.63302752 546 39.56043956 547 39.67093236 548 39.78102190 549 39.70856102 550 39.63636364 551 39.74591652 552 39.67391304 553 39.60216998 554 39.53068592 555 39.45945946 556 39.56834532 557 39.49730700 558 39.60573477 559 39.53488372 560 39.64285714 561 39.57219251 562 39.67971530 563 39.78685613 564 39.71631206 565 39.82300885 566 39.75265018 567 39.68253968 568 39.78873239 569 39.71880492 570 39.64912281 571 39.57968476 572 39.68531469 573 39.61605585 574 39.54703833 575 39.47826087 576 39.58333333 577 39.68804159 578 39.79238754 579 39.72366149 580 39.65517241 581 39.58691910 582 39.51890034 583 39.62264151 584 39.72602740 585 39.82905983 586 39.76109215 587 39.69335605 588 39.62585034 589 39.55857385 590 39.49152542 591 39.42470389 592 39.35810811 593 39.46037099 594 39.56228956 595 39.49579832 596 39.59731544 597 39.53098827 598 39.46488294 599 39.56594324 600 39.50000000 601 39.60066556 602 39.53488372 603 39.63515755 604 39.56953642 605 39.50413223 606 39.43894389 607 39.53871499 608 39.47368421 609 39.40886700 610 39.34426230 611 39.44353519 612 39.37908497 613 39.47797716 614 39.41368078 615 39.34959350 616 39.44805195 617 39.38411669 618 39.32038835 619 39.25686591 620 39.19354839 621 39.13043478 622 39.22829582 623 39.16532905 624 39.10256410 625 39.04000000 626 39.13738019 627 39.07496013 628 39.01273885 629 39.10969793 630 39.20634921 631 39.14421553 632 39.24050633 633 39.17851501 634 39.11671924 635 39.05511811 636 38.99371069 637 38.93249608 638 38.87147335 639 38.81064163 640 38.90625000 641 38.84555382 642 38.78504673 643 38.72472784 644 38.66459627 645 38.60465116 646 38.54489164 647 38.48531685 648 38.42592593 649 38.36671803 650 38.30769231 651 38.24884793 652 38.19018405 653 38.13169985 654 38.22629969 655 38.16793893 656 38.26219512 657 38.35616438 658 38.44984802 659 38.39150228 660 38.33333333 661 38.27534039 662 38.21752266 663 38.15987934 664 38.25301205 665 38.34586466 666 38.28828829 667 38.38080960 668 38.47305389 669 38.56502242 670 38.50746269 671 38.45007452 672 38.39285714 673 38.33580981 674 38.27893175 675 38.22222222 676 38.16568047 677 38.10930576 678 38.20058997 679 38.14432990 680 38.23529412 681 38.32599119 682 38.26979472 683 38.21376281 684 38.30409357 685 38.39416058 686 38.33819242 687 38.28238719 688 38.22674419 689 38.31640058 690 38.26086957 691 38.20549928 692 38.15028902 693 38.09523810 694 38.04034582 695 37.98561151 696 38.07471264 697 38.02008608 698 38.10888252 699 38.19742489 700 38.14285714 701 38.08844508 702 38.17663818 703 38.26458037 704 38.35227273 705 38.29787234 706 38.24362606 707 38.18953324 708 38.13559322 709 38.22284908 710 38.16901408 711 38.25597750 712 38.20224719 713 38.28892006 714 38.37535014 715 38.32167832 716 38.26815642 717 38.21478382 718 38.16155989 719 38.10848401 720 38.05555556 721 38.00277393 722 37.95013850 723 38.03596127 724 38.12154696 725 38.20689655 726 38.15426997 727 38.23933975 728 38.18681319 729 38.13443073 730 38.08219178 731 38.03009576 732 37.97814208 733 38.06275580 734 38.01089918 735 37.95918367 736 37.90760870 737 37.99185889 738 38.07588076 739 38.15967524 740 38.10810811 741 38.05668016 742 38.14016173 743 38.22341857 744 38.30645161 745 38.25503356 746 38.20375335 747 38.15261044 748 38.10160428 749 38.05073431 750 38.13333333 Final result: 38.1333 +/- 1.7748 Random chance: 25.0000 +/- 1.5822