common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 817 tasks in prompt multiple_choice_score: selecting 750 random tasks from 817 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 25.00000000 5 40.00000000 6 33.33333333 7 28.57142857 8 25.00000000 9 22.22222222 10 20.00000000 11 18.18181818 12 25.00000000 13 30.76923077 14 28.57142857 15 26.66666667 16 25.00000000 17 23.52941176 18 22.22222222 19 21.05263158 20 20.00000000 21 19.04761905 22 18.18181818 23 17.39130435 24 20.83333333 25 20.00000000 26 19.23076923 27 18.51851852 28 17.85714286 29 20.68965517 30 20.00000000 31 19.35483871 32 21.87500000 33 24.24242424 34 23.52941176 35 22.85714286 36 22.22222222 37 24.32432432 38 23.68421053 39 25.64102564 40 25.00000000 41 24.39024390 42 23.80952381 43 23.25581395 44 22.72727273 45 24.44444444 46 23.91304348 47 23.40425532 48 25.00000000 49 24.48979592 50 26.00000000 51 25.49019608 52 25.00000000 53 24.52830189 54 24.07407407 55 25.45454545 56 25.00000000 57 24.56140351 58 25.86206897 59 25.42372881 60 26.66666667 61 26.22950820 62 27.41935484 63 26.98412698 64 26.56250000 65 26.15384615 66 25.75757576 67 25.37313433 68 25.00000000 69 26.08695652 70 25.71428571 71 25.35211268 72 26.38888889 73 26.02739726 74 25.67567568 75 25.33333333 76 25.00000000 77 24.67532468 78 24.35897436 79 24.05063291 80 25.00000000 81 24.69135802 82 25.60975610 83 25.30120482 84 25.00000000 85 24.70588235 86 24.41860465 87 24.13793103 88 23.86363636 89 24.71910112 90 24.44444444 91 24.17582418 92 23.91304348 93 23.65591398 94 23.40425532 95 23.15789474 96 23.95833333 97 23.71134021 98 23.46938776 99 24.24242424 100 24.00000000 101 24.75247525 102 24.50980392 103 24.27184466 104 24.03846154 105 23.80952381 106 24.52830189 107 25.23364486 108 25.00000000 109 24.77064220 110 25.45454545 111 25.22522523 112 25.89285714 113 25.66371681 114 26.31578947 115 26.08695652 116 25.86206897 117 25.64102564 118 25.42372881 119 25.21008403 120 25.00000000 121 25.61983471 122 25.40983607 123 25.20325203 124 25.80645161 125 26.40000000 126 26.98412698 127 26.77165354 128 26.56250000 129 27.13178295 130 26.92307692 131 26.71755725 132 26.51515152 133 26.31578947 134 26.11940299 135 26.66666667 136 26.47058824 137 27.00729927 138 27.53623188 139 28.05755396 140 27.85714286 141 27.65957447 142 27.46478873 143 27.27272727 144 27.77777778 145 27.58620690 146 27.39726027 147 27.21088435 148 27.02702703 149 26.84563758 150 26.66666667 151 27.15231788 152 26.97368421 153 26.79738562 154 26.62337662 155 26.45161290 156 26.92307692 157 26.75159236 158 27.21518987 159 27.04402516 160 26.87500000 161 26.70807453 162 27.16049383 163 26.99386503 164 26.82926829 165 26.66666667 166 27.10843373 167 26.94610778 168 27.38095238 169 27.21893491 170 27.05882353 171 26.90058480 172 26.74418605 173 26.58959538 174 27.01149425 175 26.85714286 176 27.27272727 177 27.11864407 178 26.96629213 179 27.37430168 180 27.22222222 181 27.07182320 182 26.92307692 183 26.77595628 184 26.63043478 185 26.48648649 186 26.88172043 187 26.73796791 188 26.59574468 189 26.98412698 190 26.84210526 191 26.70157068 192 26.56250000 193 26.42487047 194 26.80412371 195 26.66666667 196 27.04081633 197 27.41116751 198 27.27272727 199 27.13567839 200 27.00000000 201 26.86567164 202 26.73267327 203 26.60098522 204 26.47058824 205 26.82926829 206 26.69902913 207 27.05314010 208 26.92307692 209 26.79425837 210 26.66666667 211 26.54028436 212 26.41509434 213 26.29107981 214 26.16822430 215 26.04651163 216 26.38888889 217 26.26728111 218 26.14678899 219 26.02739726 220 25.90909091 221 25.79185520 222 25.67567568 223 25.56053812 224 25.89285714 225 25.77777778 226 25.66371681 227 25.99118943 228 25.87719298 229 25.76419214 230 25.65217391 231 25.54112554 232 25.43103448 233 25.32188841 234 25.64102564 235 25.95744681 236 26.27118644 237 26.16033755 238 26.47058824 239 26.35983264 240 26.25000000 241 26.14107884 242 26.03305785 243 25.92592593 244 25.81967213 245 25.71428571 246 25.60975610 247 25.50607287 248 25.40322581 249 25.70281124 250 25.60000000 251 25.89641434 252 26.19047619 253 26.08695652 254 25.98425197 255 26.27450980 256 26.17187500 257 26.07003891 258 25.96899225 259 25.86872587 260 25.76923077 261 26.05363985 262 25.95419847 263 25.85551331 264 26.13636364 265 26.41509434 266 26.31578947 267 26.21722846 268 26.11940299 269 26.02230483 270 25.92592593 271 25.83025830 272 25.73529412 273 25.64102564 274 25.54744526 275 25.45454545 276 25.36231884 277 25.27075812 278 25.17985612 279 25.08960573 280 25.35714286 281 25.62277580 282 25.53191489 283 25.44169611 284 25.35211268 285 25.26315789 286 25.52447552 287 25.78397213 288 26.04166667 289 25.95155709 290 25.86206897 291 26.11683849 292 26.36986301 293 26.62116041 294 26.53061224 295 26.77966102 296 27.02702703 297 27.27272727 298 27.18120805 299 27.09030100 300 27.00000000 301 26.91029900 302 26.82119205 303 26.73267327 304 26.64473684 305 26.55737705 306 26.47058824 307 26.38436482 308 26.62337662 309 26.53721683 310 26.45161290 311 26.36655949 312 26.60256410 313 26.51757188 314 26.43312102 315 26.34920635 316 26.26582278 317 26.18296530 318 26.10062893 319 26.01880878 320 25.93750000 321 25.85669782 322 26.08695652 323 26.00619195 324 25.92592593 325 25.84615385 326 25.76687117 327 25.99388379 328 26.21951220 329 26.13981763 330 26.36363636 331 26.28398792 332 26.50602410 333 26.72672673 334 26.64670659 335 26.86567164 336 27.08333333 337 27.00296736 338 26.92307692 339 27.13864307 340 27.05882353 341 26.97947214 342 27.19298246 343 27.11370262 344 27.32558140 345 27.53623188 346 27.45664740 347 27.66570605 348 27.58620690 349 27.50716332 350 27.42857143 351 27.35042735 352 27.55681818 353 27.47875354 354 27.40112994 355 27.60563380 356 27.52808989 357 27.73109244 358 27.65363128 359 27.57660167 360 27.50000000 361 27.70083102 362 27.62430939 363 27.54820937 364 27.47252747 365 27.39726027 366 27.32240437 367 27.52043597 368 27.71739130 369 27.64227642 370 27.56756757 371 27.49326146 372 27.68817204 373 27.61394102 374 27.80748663 375 27.73333333 376 27.65957447 377 27.85145889 378 27.77777778 379 27.70448549 380 27.63157895 381 27.55905512 382 27.74869110 383 27.67624021 384 27.60416667 385 27.53246753 386 27.46113990 387 27.39018088 388 27.57731959 389 27.50642674 390 27.43589744 391 27.62148338 392 27.55102041 393 27.48091603 394 27.66497462 395 27.59493671 396 27.77777778 397 27.95969773 398 27.88944724 399 27.81954887 400 27.75000000 401 27.68079800 402 27.86069652 403 27.79156328 404 27.72277228 405 27.65432099 406 27.83251232 407 27.76412776 408 27.69607843 409 27.62836186 410 27.56097561 411 27.49391727 412 27.66990291 413 27.60290557 414 27.77777778 415 27.95180723 416 27.88461538 417 28.05755396 418 27.99043062 419 27.92362768 420 27.85714286 421 28.02850356 422 27.96208531 423 27.89598109 424 27.83018868 425 27.76470588 426 27.69953052 427 27.63466042 428 27.57009346 429 27.50582751 430 27.44186047 431 27.37819026 432 27.54629630 433 27.71362587 434 27.88018433 435 27.81609195 436 27.98165138 437 27.91762014 438 28.08219178 439 28.01822323 440 28.18181818 441 28.11791383 442 28.05429864 443 27.99097065 444 28.15315315 445 28.08988764 446 28.02690583 447 27.96420582 448 27.90178571 449 28.06236080 450 28.22222222 451 28.15964523 452 28.09734513 453 28.25607064 454 28.19383260 455 28.13186813 456 28.07017544 457 28.00875274 458 27.94759825 459 27.88671024 460 27.82608696 461 27.76572668 462 27.92207792 463 28.07775378 464 28.23275862 465 28.38709677 466 28.32618026 467 28.26552463 468 28.20512821 469 28.14498934 470 28.08510638 471 28.02547771 472 27.96610169 473 28.11839323 474 28.05907173 475 28.21052632 476 28.15126050 477 28.30188679 478 28.24267782 479 28.18371608 480 28.33333333 481 28.27442827 482 28.42323651 483 28.36438923 484 28.30578512 485 28.45360825 486 28.60082305 487 28.74743326 488 28.68852459 489 28.83435583 490 28.97959184 491 29.12423625 492 29.26829268 493 29.20892495 494 29.35222672 495 29.29292929 496 29.23387097 497 29.17505030 498 29.11646586 499 29.25851703 500 29.20000000 501 29.34131737 502 29.28286853 503 29.22465209 504 29.36507937 505 29.30693069 506 29.24901186 507 29.19132150 508 29.33070866 509 29.46954813 510 29.41176471 511 29.35420744 512 29.29687500 513 29.23976608 514 29.18287938 515 29.32038835 516 29.26356589 517 29.40038685 518 29.34362934 519 29.28709056 520 29.23076923 521 29.36660269 522 29.31034483 523 29.25430210 524 29.19847328 525 29.14285714 526 29.08745247 527 29.22201139 528 29.16666667 529 29.11153119 530 29.05660377 531 29.00188324 532 29.13533835 533 29.26829268 534 29.21348315 535 29.15887850 536 29.10447761 537 29.23649907 538 29.18215613 539 29.12801484 540 29.07407407 541 29.02033272 542 28.96678967 543 28.91344383 544 28.86029412 545 28.80733945 546 28.93772894 547 29.06764168 548 29.19708029 549 29.14389800 550 29.09090909 551 29.03811252 552 29.16666667 553 29.29475588 554 29.24187726 555 29.18918919 556 29.13669065 557 29.08438061 558 29.03225806 559 28.98032200 560 29.10714286 561 29.23351159 562 29.18149466 563 29.12966252 564 29.25531915 565 29.20353982 566 29.32862191 567 29.45326279 568 29.57746479 569 29.52548330 570 29.47368421 571 29.59719790 572 29.54545455 573 29.49389180 574 29.61672474 575 29.73913043 576 29.86111111 577 29.98266898 578 29.93079585 579 30.05181347 580 30.00000000 581 29.94836489 582 29.89690722 583 29.84562607 584 29.96575342 585 29.91452991 586 30.03412969 587 29.98296422 588 29.93197279 589 29.88115450 590 30.00000000 591 29.94923858 592 29.89864865 593 29.84822934 594 29.79797980 595 29.74789916 596 29.69798658 597 29.64824121 598 29.59866221 599 29.54924875 600 29.50000000 601 29.61730449 602 29.73421927 603 29.85074627 604 29.80132450 605 29.75206612 606 29.86798680 607 29.81878089 608 29.76973684 609 29.88505747 610 29.83606557 611 29.78723404 612 29.90196078 613 29.85318108 614 29.80456026 615 29.75609756 616 29.87012987 617 29.98379254 618 29.93527508 619 30.04846527 620 30.00000000 621 29.95169082 622 30.06430868 623 30.01605136 624 29.96794872 625 29.92000000 626 29.87220447 627 29.82456140 628 29.93630573 629 29.88871224 630 29.84126984 631 29.79397781 632 29.74683544 633 29.69984202 634 29.65299685 635 29.60629921 636 29.55974843 637 29.51334380 638 29.46708464 639 29.42097027 640 29.53125000 641 29.48517941 642 29.59501558 643 29.54898911 644 29.50310559 645 29.61240310 646 29.56656347 647 29.52086553 648 29.62962963 649 29.58397535 650 29.69230769 651 29.80030722 652 29.90797546 653 30.01531394 654 29.96941896 655 29.92366412 656 30.03048780 657 30.13698630 658 30.09118541 659 30.19726859 660 30.15151515 661 30.10590015 662 30.06042296 663 30.01508296 664 29.96987952 665 30.07518797 666 30.18018018 667 30.13493253 668 30.08982036 669 30.04484305 670 30.14925373 671 30.10432191 672 30.05952381 673 30.01485884 674 29.97032641 675 30.07407407 676 30.17751479 677 30.28064993 678 30.23598820 679 30.19145803 680 30.14705882 681 30.10279001 682 30.20527859 683 30.16105417 684 30.26315789 685 30.21897810 686 30.17492711 687 30.13100437 688 30.23255814 689 30.18867925 690 30.14492754 691 30.24602026 692 30.20231214 693 30.30303030 694 30.25936599 695 30.21582734 696 30.31609195 697 30.27259684 698 30.22922636 699 30.18597997 700 30.28571429 701 30.24251070 702 30.19943020 703 30.15647226 704 30.11363636 705 30.07092199 706 30.16997167 707 30.12729844 708 30.08474576 709 30.04231312 710 30.00000000 711 29.95780591 712 30.05617978 713 30.01402525 714 29.97198880 715 29.93006993 716 29.88826816 717 29.98605300 718 30.08356546 719 30.18080668 720 30.13888889 721 30.09708738 722 30.05540166 723 30.01383126 724 30.11049724 725 30.06896552 726 30.16528926 727 30.12379642 728 30.08241758 729 30.17832647 730 30.27397260 731 30.23255814 732 30.32786885 733 30.28649386 734 30.24523161 735 30.20408163 736 30.16304348 737 30.12211669 738 30.08130081 739 30.17591340 740 30.27027027 741 30.36437247 742 30.32345013 743 30.28263795 744 30.37634409 745 30.33557047 746 30.42895442 747 30.52208835 748 30.48128342 749 30.57409880 750 30.66666667 Final result: 30.6667 +/- 1.6849 Random chance: 19.8992 +/- 1.4588