common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 50.00000000 9 55.55555556 10 60.00000000 11 54.54545455 12 58.33333333 13 53.84615385 14 50.00000000 15 46.66666667 16 50.00000000 17 52.94117647 18 50.00000000 19 52.63157895 20 55.00000000 21 57.14285714 22 54.54545455 23 52.17391304 24 54.16666667 25 56.00000000 26 53.84615385 27 55.55555556 28 53.57142857 29 55.17241379 30 56.66666667 31 58.06451613 32 59.37500000 33 57.57575758 34 58.82352941 35 60.00000000 36 58.33333333 37 56.75675676 38 57.89473684 39 56.41025641 40 55.00000000 41 53.65853659 42 52.38095238 43 53.48837209 44 52.27272727 45 53.33333333 46 52.17391304 47 51.06382979 48 52.08333333 49 53.06122449 50 52.00000000 51 50.98039216 52 51.92307692 53 50.94339623 54 50.00000000 55 50.90909091 56 51.78571429 57 50.87719298 58 50.00000000 59 50.84745763 60 50.00000000 61 50.81967213 62 50.00000000 63 49.20634921 64 48.43750000 65 47.69230769 66 46.96969697 67 46.26865672 68 45.58823529 69 44.92753623 70 45.71428571 71 46.47887324 72 47.22222222 73 46.57534247 74 47.29729730 75 48.00000000 76 48.68421053 77 49.35064935 78 50.00000000 79 49.36708861 80 48.75000000 81 49.38271605 82 48.78048780 83 49.39759036 84 50.00000000 85 49.41176471 86 48.83720930 87 49.42528736 88 50.00000000 89 49.43820225 90 48.88888889 91 48.35164835 92 47.82608696 93 47.31182796 94 47.87234043 95 47.36842105 96 47.91666667 97 47.42268041 98 46.93877551 99 46.46464646 100 46.00000000 101 46.53465347 102 46.07843137 103 45.63106796 104 46.15384615 105 46.66666667 106 46.22641509 107 45.79439252 108 46.29629630 109 46.78899083 110 47.27272727 111 47.74774775 112 47.32142857 113 46.90265487 114 47.36842105 115 47.82608696 116 47.41379310 117 47.86324786 118 48.30508475 119 47.89915966 120 47.50000000 121 47.93388430 122 47.54098361 123 47.15447154 124 46.77419355 125 47.20000000 126 46.82539683 127 46.45669291 128 46.09375000 129 45.73643411 130 45.38461538 131 45.03816794 132 44.69696970 133 44.36090226 134 44.02985075 135 43.70370370 136 43.38235294 137 43.79562044 138 43.47826087 139 43.88489209 140 43.57142857 141 43.26241135 142 43.66197183 143 44.05594406 144 44.44444444 145 44.82758621 146 45.20547945 147 44.89795918 148 45.27027027 149 44.96644295 150 44.66666667 151 44.37086093 152 44.73684211 153 45.09803922 154 44.80519481 155 44.51612903 156 44.23076923 157 44.58598726 158 44.93670886 159 44.65408805 160 45.00000000 161 44.72049689 162 44.44444444 163 44.17177914 164 43.90243902 165 44.24242424 166 43.97590361 167 44.31137725 168 44.04761905 169 44.37869822 170 44.11764706 171 44.44444444 172 44.18604651 173 43.93063584 174 43.67816092 175 43.42857143 176 43.18181818 177 42.93785311 178 42.69662921 179 43.01675978 180 43.33333333 181 43.64640884 182 43.40659341 183 43.71584699 184 43.47826087 185 43.78378378 186 44.08602151 187 43.85026738 188 44.14893617 189 44.44444444 190 44.21052632 191 44.50261780 192 44.27083333 193 44.04145078 194 44.32989691 195 44.61538462 196 44.89795918 197 44.67005076 198 44.44444444 199 44.22110553 200 44.00000000 201 43.78109453 202 43.56435644 203 43.34975369 204 43.13725490 205 43.41463415 206 43.20388350 207 43.47826087 208 43.26923077 209 43.06220096 210 42.85714286 211 43.12796209 212 42.92452830 213 42.72300469 214 42.52336449 215 42.79069767 216 43.05555556 217 42.85714286 218 43.11926606 219 43.37899543 220 43.63636364 221 43.43891403 222 43.69369369 223 43.94618834 224 43.75000000 225 44.00000000 226 43.80530973 227 43.61233480 228 43.85964912 229 43.66812227 230 43.47826087 231 43.29004329 232 43.53448276 233 43.34763948 234 43.58974359 235 43.40425532 236 43.22033898 237 43.03797468 238 42.85714286 239 42.67782427 240 42.91666667 241 42.73858921 242 42.97520661 243 43.20987654 244 43.03278689 245 43.26530612 246 43.08943089 247 42.91497976 248 43.14516129 249 42.97188755 250 42.80000000 251 42.62948207 252 42.46031746 253 42.29249012 254 42.12598425 255 41.96078431 256 42.18750000 257 42.41245136 258 42.24806202 259 42.08494208 260 42.30769231 261 42.14559387 262 41.98473282 263 41.82509506 264 41.66666667 265 41.88679245 266 42.10526316 267 41.94756554 268 41.79104478 269 42.00743494 270 41.85185185 271 41.69741697 272 41.91176471 273 41.75824176 274 41.60583942 275 41.45454545 276 41.66666667 277 41.87725632 278 41.72661871 279 41.57706093 280 41.78571429 281 41.63701068 282 41.84397163 283 41.69611307 284 41.54929577 285 41.40350877 286 41.25874126 287 41.46341463 288 41.31944444 289 41.17647059 290 41.37931034 291 41.58075601 292 41.78082192 293 41.63822526 294 41.49659864 295 41.35593220 296 41.21621622 297 41.07744108 298 41.27516779 299 41.13712375 300 41.33333333 301 41.19601329 302 41.05960265 303 40.92409241 304 40.78947368 305 40.98360656 306 40.84967320 307 41.04234528 308 41.23376623 309 41.10032362 310 40.96774194 311 40.83601286 312 41.02564103 313 40.89456869 314 40.76433121 315 40.63492063 316 40.50632911 317 40.69400631 318 40.88050314 319 41.06583072 320 40.93750000 321 40.80996885 322 40.68322981 323 40.55727554 324 40.43209877 325 40.61538462 326 40.49079755 327 40.67278287 328 40.54878049 329 40.42553191 330 40.30303030 331 40.18126888 332 40.36144578 333 40.54054054 334 40.41916168 335 40.59701493 336 40.77380952 337 40.65281899 338 40.53254438 339 40.41297935 340 40.58823529 341 40.76246334 342 40.93567251 343 40.81632653 344 40.69767442 345 40.57971014 346 40.75144509 347 40.92219020 348 40.80459770 349 40.68767908 350 40.85714286 351 40.74074074 352 40.62500000 353 40.79320113 354 40.67796610 355 40.56338028 356 40.73033708 357 40.89635854 358 40.78212291 359 40.66852368 360 40.83333333 361 40.72022161 362 40.88397790 363 41.04683196 364 40.93406593 365 40.82191781 366 40.98360656 367 40.87193460 368 40.76086957 369 40.65040650 370 40.81081081 371 40.70080863 372 40.59139785 373 40.48257373 374 40.37433155 375 40.53333333 376 40.42553191 377 40.31830239 378 40.47619048 379 40.36939314 380 40.26315789 381 40.41994751 382 40.57591623 383 40.73107050 384 40.62500000 385 40.77922078 386 40.67357513 387 40.56847545 388 40.46391753 389 40.61696658 390 40.76923077 391 40.92071611 392 41.07142857 393 40.96692112 394 40.86294416 395 41.01265823 396 41.16161616 397 41.30982368 398 41.20603015 399 41.35338346 400 41.25000000 401 41.39650873 402 41.54228856 403 41.43920596 404 41.33663366 405 41.23456790 406 41.13300493 407 41.03194103 408 40.93137255 409 41.07579462 410 40.97560976 411 40.87591241 412 40.77669903 413 40.67796610 414 40.82125604 415 40.72289157 416 40.86538462 417 40.76738609 418 40.66985646 419 40.81145585 420 40.71428571 421 40.85510689 422 40.75829384 423 40.66193853 424 40.80188679 425 40.70588235 426 40.61032864 427 40.74941452 428 40.65420561 429 40.79254079 430 40.69767442 431 40.83526682 432 40.74074074 433 40.64665127 434 40.55299539 435 40.45977011 436 40.36697248 437 40.27459954 438 40.41095890 439 40.54669704 440 40.45454545 441 40.58956916 442 40.49773756 443 40.40632054 444 40.31531532 445 40.22471910 446 40.13452915 447 40.04474273 448 39.95535714 449 39.86636971 450 40.00000000 451 40.13303769 452 40.04424779 453 40.17660044 454 40.08810573 455 40.00000000 456 40.13157895 457 40.04376368 458 39.95633188 459 40.08714597 460 40.21739130 461 40.13015184 462 40.25974026 463 40.17278618 464 40.30172414 465 40.21505376 466 40.12875536 467 40.25695931 468 40.38461538 469 40.51172708 470 40.42553191 471 40.33970276 472 40.25423729 473 40.38054968 474 40.29535865 475 40.21052632 476 40.33613445 477 40.25157233 478 40.37656904 479 40.29227557 480 40.41666667 481 40.33264033 482 40.24896266 483 40.37267081 484 40.28925620 485 40.20618557 486 40.12345679 487 40.24640657 488 40.16393443 489 40.08179959 490 40.20408163 491 40.12219959 492 40.04065041 493 39.95943205 494 40.08097166 495 40.20202020 496 40.32258065 497 40.24144869 498 40.16064257 499 40.28056112 500 40.40000000 501 40.31936128 502 40.23904382 503 40.35785288 504 40.27777778 505 40.19801980 506 40.31620553 507 40.23668639 508 40.15748031 509 40.07858546 510 40.19607843 511 40.11741683 512 40.03906250 513 39.96101365 514 39.88326848 515 39.80582524 516 39.92248062 517 40.03868472 518 40.15444015 519 40.07707129 520 40.00000000 521 39.92322457 522 40.03831418 523 39.96175908 524 40.07633588 525 40.00000000 526 39.92395437 527 40.03795066 528 40.15151515 529 40.07561437 530 40.18867925 531 40.11299435 532 40.03759398 533 39.96247655 534 39.88764045 535 40.00000000 536 39.92537313 537 40.03724395 538 40.14869888 539 40.07421150 540 40.00000000 541 39.92606285 542 40.03690037 543 39.96316759 544 40.07352941 545 40.00000000 546 39.92673993 547 40.03656307 548 40.14598540 549 40.07285974 550 40.00000000 551 40.10889292 552 40.03623188 553 39.96383363 554 39.89169675 555 39.81981982 556 39.92805755 557 39.85637343 558 39.96415771 559 40.07155635 560 40.17857143 561 40.10695187 562 40.21352313 563 40.31971581 564 40.24822695 565 40.35398230 566 40.28268551 567 40.21164021 568 40.31690141 569 40.24604569 570 40.17543860 571 40.10507881 572 40.20979021 573 40.13961606 574 40.06968641 575 40.00000000 576 40.10416667 577 40.20797227 578 40.31141869 579 40.24179620 580 40.17241379 581 40.10327022 582 40.03436426 583 40.13722127 584 40.23972603 585 40.34188034 586 40.27303754 587 40.20442930 588 40.13605442 589 40.06791171 590 40.00000000 591 39.93231810 592 39.86486486 593 39.96627319 594 40.06734007 595 40.00000000 596 40.10067114 597 40.03350084 598 39.96655518 599 40.06677796 600 40.00000000 601 39.93344426 602 39.86710963 603 39.96683250 604 39.90066225 605 39.83471074 606 39.76897690 607 39.86820428 608 39.80263158 609 39.73727422 610 39.67213115 611 39.77086743 612 39.70588235 613 39.80424144 614 39.90228013 615 39.83739837 616 39.93506494 617 39.87034036 618 39.80582524 619 39.74151858 620 39.67741935 621 39.61352657 622 39.71061093 623 39.64686998 624 39.58333333 625 39.52000000 626 39.61661342 627 39.55342903 628 39.49044586 629 39.58664547 630 39.52380952 631 39.46117274 632 39.55696203 633 39.49447077 634 39.43217666 635 39.37007874 636 39.30817610 637 39.40345369 638 39.34169279 639 39.28012520 640 39.37500000 641 39.31357254 642 39.25233645 643 39.19129082 644 39.28571429 645 39.22480620 646 39.16408669 647 39.10355487 648 39.04320988 649 38.98305085 650 38.92307692 651 38.86328725 652 38.80368098 653 38.74425727 654 38.83792049 655 38.77862595 656 38.87195122 657 38.81278539 658 38.90577508 659 38.84673748 660 38.78787879 661 38.72919818 662 38.67069486 663 38.61236802 664 38.70481928 665 38.79699248 666 38.73873874 667 38.68065967 668 38.77245509 669 38.86397608 670 38.80597015 671 38.74813711 672 38.69047619 673 38.63298663 674 38.57566766 675 38.51851852 676 38.46153846 677 38.40472674 678 38.49557522 679 38.43888071 680 38.52941176 681 38.61967695 682 38.56304985 683 38.50658858 684 38.59649123 685 38.54014599 686 38.48396501 687 38.42794760 688 38.37209302 689 38.46153846 690 38.40579710 691 38.35021708 692 38.29479769 693 38.23953824 694 38.18443804 695 38.12949640 696 38.21839080 697 38.16355811 698 38.25214900 699 38.34048641 700 38.28571429 701 38.23109843 702 38.31908832 703 38.26458037 704 38.35227273 705 38.29787234 706 38.24362606 707 38.18953324 708 38.13559322 709 38.08180536 710 38.02816901 711 38.11533052 712 38.06179775 713 38.14866760 714 38.23529412 715 38.18181818 716 38.12849162 717 38.07531381 718 38.02228412 719 37.96940195 720 37.91666667 721 37.86407767 722 37.81163435 723 37.89764869 724 37.98342541 725 38.06896552 726 38.01652893 727 38.10178817 728 38.04945055 729 37.99725652 730 37.94520548 731 37.89329685 732 37.84153005 733 37.92633015 734 37.87465940 735 37.82312925 736 37.77173913 737 37.85617368 738 37.94037940 739 38.02435724 740 37.97297297 741 37.92172740 742 38.00539084 743 38.08882907 744 38.17204301 745 38.12080537 746 38.06970509 747 38.01874163 748 37.96791444 749 37.91722296 750 38.00000000 Final result: 38.0000 +/- 1.7736 Random chance: 25.0000 +/- 1.5822