common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 869 tasks in prompt multiple_choice_score: selecting 750 random tasks from 869 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 66.66666667 4 50.00000000 5 60.00000000 6 66.66666667 7 57.14285714 8 62.50000000 9 55.55555556 10 50.00000000 11 45.45454545 12 41.66666667 13 38.46153846 14 42.85714286 15 46.66666667 16 43.75000000 17 47.05882353 18 44.44444444 19 47.36842105 20 45.00000000 21 47.61904762 22 50.00000000 23 47.82608696 24 50.00000000 25 48.00000000 26 46.15384615 27 48.14814815 28 46.42857143 29 44.82758621 30 46.66666667 31 48.38709677 32 46.87500000 33 48.48484848 34 47.05882353 35 45.71428571 36 44.44444444 37 45.94594595 38 47.36842105 39 46.15384615 40 47.50000000 41 48.78048780 42 47.61904762 43 48.83720930 44 47.72727273 45 46.66666667 46 47.82608696 47 48.93617021 48 47.91666667 49 48.97959184 50 50.00000000 51 49.01960784 52 50.00000000 53 50.94339623 54 50.00000000 55 49.09090909 56 50.00000000 57 50.87719298 58 51.72413793 59 50.84745763 60 51.66666667 61 52.45901639 62 51.61290323 63 52.38095238 64 51.56250000 65 50.76923077 66 51.51515152 67 50.74626866 68 51.47058824 69 52.17391304 70 51.42857143 71 50.70422535 72 50.00000000 73 50.68493151 74 51.35135135 75 52.00000000 76 52.63157895 77 53.24675325 78 53.84615385 79 54.43037975 80 53.75000000 81 53.08641975 82 53.65853659 83 53.01204819 84 53.57142857 85 54.11764706 86 53.48837209 87 54.02298851 88 54.54545455 89 55.05617978 90 55.55555556 91 56.04395604 92 56.52173913 93 55.91397849 94 55.31914894 95 55.78947368 96 55.20833333 97 54.63917526 98 55.10204082 99 55.55555556 100 55.00000000 101 54.45544554 102 54.90196078 103 54.36893204 104 54.80769231 105 55.23809524 106 54.71698113 107 55.14018692 108 54.62962963 109 54.12844037 110 54.54545455 111 54.95495495 112 55.35714286 113 54.86725664 114 54.38596491 115 54.78260870 116 54.31034483 117 54.70085470 118 55.08474576 119 54.62184874 120 55.00000000 121 54.54545455 122 54.91803279 123 55.28455285 124 54.83870968 125 55.20000000 126 55.55555556 127 55.90551181 128 55.46875000 129 55.03875969 130 54.61538462 131 54.19847328 132 53.78787879 133 54.13533835 134 53.73134328 135 53.33333333 136 52.94117647 137 53.28467153 138 53.62318841 139 53.95683453 140 53.57142857 141 53.90070922 142 54.22535211 143 54.54545455 144 54.16666667 145 54.48275862 146 54.10958904 147 54.42176871 148 54.05405405 149 54.36241611 150 54.66666667 151 54.96688742 152 55.26315789 153 54.90196078 154 54.54545455 155 54.19354839 156 54.48717949 157 54.77707006 158 55.06329114 159 55.34591195 160 55.00000000 161 55.27950311 162 55.55555556 163 55.21472393 164 54.87804878 165 54.54545455 166 54.81927711 167 54.49101796 168 54.76190476 169 54.43786982 170 54.70588235 171 54.97076023 172 54.65116279 173 54.91329480 174 54.59770115 175 54.28571429 176 53.97727273 177 54.23728814 178 53.93258427 179 54.18994413 180 53.88888889 181 53.59116022 182 53.84615385 183 53.55191257 184 53.26086957 185 53.51351351 186 53.76344086 187 54.01069519 188 53.72340426 189 53.43915344 190 53.15789474 191 53.40314136 192 53.12500000 193 52.84974093 194 53.09278351 195 53.33333333 196 53.57142857 197 53.80710660 198 54.04040404 199 54.27135678 200 54.50000000 201 54.72636816 202 54.95049505 203 54.67980296 204 54.90196078 205 54.63414634 206 54.36893204 207 54.58937198 208 54.80769231 209 55.02392344 210 55.23809524 211 55.45023697 212 55.18867925 213 55.39906103 214 55.60747664 215 55.81395349 216 55.55555556 217 55.29953917 218 55.50458716 219 55.70776256 220 55.45454545 221 55.20361991 222 54.95495495 223 54.70852018 224 54.91071429 225 54.66666667 226 54.42477876 227 54.62555066 228 54.82456140 229 55.02183406 230 54.78260870 231 54.54545455 232 54.31034483 233 54.50643777 234 54.27350427 235 54.04255319 236 54.23728814 237 54.00843882 238 54.20168067 239 54.39330544 240 54.58333333 241 54.77178423 242 54.54545455 243 54.32098765 244 54.09836066 245 53.87755102 246 53.65853659 247 53.84615385 248 53.62903226 249 53.81526104 250 54.00000000 251 54.18326693 252 54.36507937 253 54.15019763 254 53.93700787 255 54.11764706 256 54.29687500 257 54.47470817 258 54.65116279 259 54.44015444 260 54.61538462 261 54.78927203 262 54.96183206 263 55.13307985 264 55.30303030 265 55.47169811 266 55.26315789 267 55.43071161 268 55.22388060 269 55.39033457 270 55.55555556 271 55.35055351 272 55.14705882 273 55.31135531 274 55.10948905 275 54.90909091 276 55.07246377 277 55.23465704 278 55.03597122 279 54.83870968 280 55.00000000 281 55.16014235 282 55.31914894 283 55.12367491 284 55.28169014 285 55.08771930 286 55.24475524 287 55.40069686 288 55.55555556 289 55.36332180 290 55.17241379 291 55.32646048 292 55.13698630 293 55.29010239 294 55.10204082 295 55.25423729 296 55.06756757 297 55.21885522 298 55.03355705 299 55.18394649 300 55.00000000 301 54.81727575 302 54.96688742 303 55.11551155 304 54.93421053 305 55.08196721 306 54.90196078 307 55.04885993 308 55.19480519 309 55.33980583 310 55.48387097 311 55.62700965 312 55.76923077 313 55.91054313 314 55.73248408 315 55.55555556 316 55.69620253 317 55.83596215 318 55.66037736 319 55.48589342 320 55.31250000 321 55.14018692 322 55.27950311 323 55.10835913 324 54.93827160 325 54.76923077 326 54.90797546 327 54.74006116 328 54.57317073 329 54.71124620 330 54.84848485 331 54.98489426 332 54.81927711 333 54.95495495 334 54.79041916 335 54.62686567 336 54.76190476 337 54.59940653 338 54.43786982 339 54.57227139 340 54.70588235 341 54.83870968 342 54.67836257 343 54.81049563 344 54.94186047 345 54.78260870 346 54.91329480 347 54.75504323 348 54.59770115 349 54.72779370 350 54.57142857 351 54.70085470 352 54.54545455 353 54.67422096 354 54.51977401 355 54.36619718 356 54.21348315 357 54.06162465 358 54.18994413 359 54.03899721 360 54.16666667 361 54.29362881 362 54.14364641 363 54.26997245 364 54.12087912 365 53.97260274 366 53.82513661 367 53.95095368 368 54.07608696 369 54.20054201 370 54.05405405 371 53.90835580 372 54.03225806 373 53.88739946 374 54.01069519 375 53.86666667 376 53.98936170 377 53.84615385 378 53.96825397 379 53.82585752 380 53.94736842 381 53.80577428 382 53.66492147 383 53.52480418 384 53.38541667 385 53.24675325 386 53.36787565 387 53.22997416 388 53.09278351 389 52.95629820 390 53.07692308 391 52.94117647 392 53.06122449 393 53.18066158 394 53.29949239 395 53.41772152 396 53.28282828 397 53.40050378 398 53.26633166 399 53.13283208 400 53.25000000 401 53.11720698 402 53.23383085 403 53.34987593 404 53.46534653 405 53.58024691 406 53.69458128 407 53.80835381 408 53.92156863 409 53.78973105 410 53.90243902 411 54.01459854 412 54.12621359 413 54.23728814 414 54.34782609 415 54.21686747 416 54.08653846 417 53.95683453 418 53.82775120 419 53.69928401 420 53.80952381 421 53.91923990 422 54.02843602 423 54.13711584 424 54.00943396 425 54.11764706 426 53.99061033 427 53.86416862 428 53.73831776 429 53.84615385 430 53.72093023 431 53.82830626 432 53.70370370 433 53.57967667 434 53.45622120 435 53.56321839 436 53.66972477 437 53.77574371 438 53.65296804 439 53.75854214 440 53.63636364 441 53.74149660 442 53.61990950 443 53.49887133 444 53.37837838 445 53.48314607 446 53.58744395 447 53.69127517 448 53.79464286 449 53.89755011 450 53.77777778 451 53.88026608 452 53.98230088 453 53.86313466 454 53.74449339 455 53.84615385 456 53.72807018 457 53.61050328 458 53.49344978 459 53.59477124 460 53.47826087 461 53.57917570 462 53.67965368 463 53.56371490 464 53.44827586 465 53.33333333 466 53.43347639 467 53.31905782 468 53.20512821 469 53.09168443 470 53.19148936 471 53.29087049 472 53.17796610 473 53.06553911 474 52.95358650 475 52.84210526 476 52.73109244 477 52.83018868 478 52.92887029 479 53.02713987 480 52.91666667 481 52.80665281 482 52.90456432 483 53.00207039 484 53.09917355 485 53.19587629 486 53.29218107 487 53.18275154 488 53.27868852 489 53.16973415 490 53.06122449 491 53.15682281 492 53.04878049 493 52.94117647 494 53.03643725 495 52.92929293 496 52.82258065 497 52.91750503 498 53.01204819 499 53.10621242 500 53.20000000 501 53.29341317 502 53.38645418 503 53.47912525 504 53.57142857 505 53.66336634 506 53.55731225 507 53.45167653 508 53.54330709 509 53.43811395 510 53.52941176 511 53.42465753 512 53.51562500 513 53.41130604 514 53.50194553 515 53.59223301 516 53.48837209 517 53.38491296 518 53.28185328 519 53.17919075 520 53.07692308 521 53.16698656 522 53.25670498 523 53.15487572 524 53.05343511 525 52.95238095 526 52.85171103 527 52.75142315 528 52.65151515 529 52.55198488 530 52.64150943 531 52.73069680 532 52.81954887 533 52.90806754 534 52.80898876 535 52.71028037 536 52.61194030 537 52.70018622 538 52.60223048 539 52.69016698 540 52.59259259 541 52.49537893 542 52.39852399 543 52.48618785 544 52.38970588 545 52.47706422 546 52.38095238 547 52.28519196 548 52.37226277 549 52.45901639 550 52.36363636 551 52.26860254 552 52.17391304 553 52.26039783 554 52.16606498 555 52.25225225 556 52.15827338 557 52.24416517 558 52.15053763 559 52.23613596 560 52.32142857 561 52.40641711 562 52.49110320 563 52.57548845 564 52.48226950 565 52.38938053 566 52.47349823 567 52.55731922 568 52.64084507 569 52.54833040 570 52.45614035 571 52.36427320 572 52.27272727 573 52.18150087 574 52.26480836 575 52.34782609 576 52.43055556 577 52.51299827 578 52.42214533 579 52.50431779 580 52.58620690 581 52.49569707 582 52.40549828 583 52.48713551 584 52.56849315 585 52.64957265 586 52.55972696 587 52.47018739 588 52.38095238 589 52.46179966 590 52.54237288 591 52.62267343 592 52.70270270 593 52.61382799 594 52.69360269 595 52.77310924 596 52.68456376 597 52.59631491 598 52.50836120 599 52.42070117 600 52.33333333 601 52.41264559 602 52.32558140 603 52.23880597 604 52.15231788 605 52.23140496 606 52.31023102 607 52.22405272 608 52.30263158 609 52.21674877 610 52.29508197 611 52.20949264 612 52.12418301 613 52.03915171 614 52.11726384 615 52.19512195 616 52.11038961 617 52.18800648 618 52.10355987 619 52.01938611 620 52.09677419 621 52.17391304 622 52.25080386 623 52.32744783 624 52.24358974 625 52.16000000 626 52.07667732 627 52.15311005 628 52.22929936 629 52.30524642 630 52.38095238 631 52.45641838 632 52.37341772 633 52.29067930 634 52.20820189 635 52.28346457 636 52.35849057 637 52.27629513 638 52.35109718 639 52.42566510 640 52.34375000 641 52.41809672 642 52.33644860 643 52.25505443 644 52.32919255 645 52.24806202 646 52.16718266 647 52.08655332 648 52.00617284 649 51.92604006 650 51.84615385 651 51.92012289 652 51.84049080 653 51.76110260 654 51.83486239 655 51.75572519 656 51.67682927 657 51.59817352 658 51.67173252 659 51.59332322 660 51.51515152 661 51.43721634 662 51.51057402 663 51.58371041 664 51.65662651 665 51.72932331 666 51.80180180 667 51.87406297 668 51.79640719 669 51.71898356 670 51.79104478 671 51.71385991 672 51.63690476 673 51.56017831 674 51.48367953 675 51.40740741 676 51.47928994 677 51.55096012 678 51.62241888 679 51.69366716 680 51.61764706 681 51.54185022 682 51.61290323 683 51.53733529 684 51.60818713 685 51.67883212 686 51.60349854 687 51.67394469 688 51.74418605 689 51.66908563 690 51.73913043 691 51.66425470 692 51.73410405 693 51.80375180 694 51.87319885 695 51.94244604 696 51.86781609 697 51.93687231 698 51.86246418 699 51.78826896 700 51.71428571 701 51.64051355 702 51.56695157 703 51.49359886 704 51.56250000 705 51.63120567 706 51.55807365 707 51.62659123 708 51.69491525 709 51.76304654 710 51.69014085 711 51.75808720 712 51.82584270 713 51.89340813 714 51.82072829 715 51.74825175 716 51.67597765 717 51.60390516 718 51.53203343 719 51.46036161 720 51.52777778 721 51.45631068 722 51.52354571 723 51.59059474 724 51.65745856 725 51.72413793 726 51.65289256 727 51.58184319 728 51.64835165 729 51.71467764 730 51.78082192 731 51.84678523 732 51.91256831 733 51.84174625 734 51.90735695 735 51.83673469 736 51.90217391 737 51.83175034 738 51.76151762 739 51.69147497 740 51.62162162 741 51.68690958 742 51.75202156 743 51.81695828 744 51.74731183 745 51.81208054 746 51.74262735 747 51.67336011 748 51.60427807 749 51.53538051 750 51.60000000 Final result: 51.6000 ±1.8260 Random chance: 25.0083 ±1.5824