common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 869 tasks in prompt multiple_choice_score: selecting 750 random tasks from 869 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 100.00000000 3 100.00000000 4 75.00000000 5 80.00000000 6 83.33333333 7 85.71428571 8 87.50000000 9 77.77777778 10 80.00000000 11 72.72727273 12 75.00000000 13 76.92307692 14 78.57142857 15 80.00000000 16 81.25000000 17 82.35294118 18 77.77777778 19 78.94736842 20 75.00000000 21 76.19047619 22 77.27272727 23 73.91304348 24 75.00000000 25 72.00000000 26 69.23076923 27 70.37037037 28 67.85714286 29 68.96551724 30 70.00000000 31 67.74193548 32 65.62500000 33 63.63636364 34 61.76470588 35 60.00000000 36 58.33333333 37 59.45945946 38 57.89473684 39 58.97435897 40 60.00000000 41 60.97560976 42 59.52380952 43 58.13953488 44 56.81818182 45 57.77777778 46 58.69565217 47 59.57446809 48 60.41666667 49 61.22448980 50 60.00000000 51 60.78431373 52 61.53846154 53 62.26415094 54 61.11111111 55 60.00000000 56 60.71428571 57 61.40350877 58 62.06896552 59 62.71186441 60 63.33333333 61 62.29508197 62 61.29032258 63 61.90476190 64 62.50000000 65 61.53846154 66 62.12121212 67 62.68656716 68 63.23529412 69 63.76811594 70 62.85714286 71 61.97183099 72 61.11111111 73 61.64383562 74 60.81081081 75 61.33333333 76 60.52631579 77 59.74025974 78 60.25641026 79 59.49367089 80 60.00000000 81 60.49382716 82 60.97560976 83 60.24096386 84 60.71428571 85 61.17647059 86 61.62790698 87 60.91954023 88 61.36363636 89 61.79775281 90 62.22222222 91 62.63736264 92 63.04347826 93 62.36559140 94 61.70212766 95 62.10526316 96 62.50000000 97 61.85567010 98 62.24489796 99 62.62626263 100 63.00000000 101 63.36633663 102 63.72549020 103 63.10679612 104 62.50000000 105 62.85714286 106 62.26415094 107 62.61682243 108 62.03703704 109 61.46788991 110 61.81818182 111 62.16216216 112 62.50000000 113 62.83185841 114 63.15789474 115 63.47826087 116 62.93103448 117 63.24786325 118 63.55932203 119 63.02521008 120 63.33333333 121 62.80991736 122 63.11475410 123 63.41463415 124 62.90322581 125 63.20000000 126 63.49206349 127 63.77952756 128 64.06250000 129 63.56589147 130 63.84615385 131 64.12213740 132 64.39393939 133 64.66165414 134 64.17910448 135 63.70370370 136 63.97058824 137 64.23357664 138 64.49275362 139 64.74820144 140 65.00000000 141 64.53900709 142 64.78873239 143 65.03496503 144 65.27777778 145 65.51724138 146 65.06849315 147 64.62585034 148 64.86486486 149 65.10067114 150 65.33333333 151 65.56291391 152 65.78947368 153 66.01307190 154 65.58441558 155 65.80645161 156 66.02564103 157 66.24203822 158 65.82278481 159 66.03773585 160 66.25000000 161 66.45962733 162 66.66666667 163 66.87116564 164 66.46341463 165 66.06060606 166 66.26506024 167 65.86826347 168 66.07142857 169 65.68047337 170 65.88235294 171 66.08187135 172 66.27906977 173 66.47398844 174 66.66666667 175 66.28571429 176 66.47727273 177 66.66666667 178 66.29213483 179 66.48044693 180 66.66666667 181 66.85082873 182 67.03296703 183 66.66666667 184 66.84782609 185 67.02702703 186 67.20430108 187 67.37967914 188 67.55319149 189 67.19576720 190 66.84210526 191 67.01570681 192 67.18750000 193 66.83937824 194 67.01030928 195 67.17948718 196 66.83673469 197 67.00507614 198 67.17171717 199 66.83417085 200 67.00000000 201 67.16417910 202 67.32673267 203 67.48768473 204 67.64705882 205 67.31707317 206 66.99029126 207 67.14975845 208 67.30769231 209 67.46411483 210 67.61904762 211 67.77251185 212 67.45283019 213 67.60563380 214 67.75700935 215 67.90697674 216 67.59259259 217 67.74193548 218 67.43119266 219 67.57990868 220 67.72727273 221 67.42081448 222 67.56756757 223 67.26457399 224 66.96428571 225 66.66666667 226 66.37168142 227 66.51982379 228 66.66666667 229 66.81222707 230 66.95652174 231 66.66666667 232 66.37931034 233 66.52360515 234 66.66666667 235 66.38297872 236 66.52542373 237 66.66666667 238 66.80672269 239 66.94560669 240 67.08333333 241 67.21991701 242 67.35537190 243 67.48971193 244 67.21311475 245 67.34693878 246 67.47967480 247 67.61133603 248 67.74193548 249 67.87148594 250 68.00000000 251 68.12749004 252 68.25396825 253 68.37944664 254 68.11023622 255 68.23529412 256 67.96875000 257 67.70428016 258 67.82945736 259 67.56756757 260 67.69230769 261 67.81609195 262 67.93893130 263 68.06083650 264 68.18181818 265 67.92452830 266 67.66917293 267 67.79026217 268 67.91044776 269 68.02973978 270 68.14814815 271 67.89667897 272 67.64705882 273 67.39926740 274 67.15328467 275 67.27272727 276 67.39130435 277 67.50902527 278 67.26618705 279 67.38351254 280 67.50000000 281 67.61565836 282 67.73049645 283 67.49116608 284 67.60563380 285 67.71929825 286 67.83216783 287 67.94425087 288 68.05555556 289 67.82006920 290 67.93103448 291 68.04123711 292 68.15068493 293 68.25938567 294 68.02721088 295 68.13559322 296 68.24324324 297 68.35016835 298 68.45637584 299 68.56187291 300 68.33333333 301 68.10631229 302 68.21192053 303 68.31683168 304 68.42105263 305 68.19672131 306 67.97385621 307 68.07817590 308 68.18181818 309 68.28478964 310 68.38709677 311 68.48874598 312 68.26923077 313 68.37060703 314 68.15286624 315 67.93650794 316 68.03797468 317 68.13880126 318 67.92452830 319 67.71159875 320 67.50000000 321 67.60124611 322 67.70186335 323 67.80185759 324 67.59259259 325 67.69230769 326 67.79141104 327 67.58409786 328 67.68292683 329 67.78115502 330 67.57575758 331 67.67371601 332 67.77108434 333 67.86786787 334 67.66467066 335 67.46268657 336 67.26190476 337 67.35905045 338 67.15976331 339 67.25663717 340 67.35294118 341 67.44868035 342 67.25146199 343 67.34693878 344 67.44186047 345 67.24637681 346 67.05202312 347 66.85878963 348 66.66666667 349 66.76217765 350 66.85714286 351 66.95156695 352 67.04545455 353 66.85552408 354 66.66666667 355 66.47887324 356 66.57303371 357 66.66666667 358 66.75977654 359 66.57381616 360 66.66666667 361 66.75900277 362 66.85082873 363 66.94214876 364 67.03296703 365 67.12328767 366 66.93989071 367 66.75749319 368 66.84782609 369 66.93766938 370 67.02702703 371 66.84636119 372 66.93548387 373 67.02412869 374 67.11229947 375 66.93333333 376 67.02127660 377 67.10875332 378 67.19576720 379 67.01846966 380 67.10526316 381 67.19160105 382 67.01570681 383 66.84073107 384 66.92708333 385 66.75324675 386 66.83937824 387 66.92506460 388 67.01030928 389 66.83804627 390 66.92307692 391 66.75191816 392 66.83673469 393 66.92111959 394 67.00507614 395 67.08860759 396 66.91919192 397 67.00251889 398 66.83417085 399 66.91729323 400 67.00000000 401 66.83291771 402 66.91542289 403 66.99751861 404 67.07920792 405 67.16049383 406 67.24137931 407 67.32186732 408 67.40196078 409 67.23716381 410 67.31707317 411 67.39659367 412 67.47572816 413 67.55447942 414 67.63285024 415 67.46987952 416 67.30769231 417 67.38609113 418 67.46411483 419 67.30310263 420 67.38095238 421 67.22090261 422 67.06161137 423 67.13947991 424 66.98113208 425 67.05882353 426 67.13615023 427 67.21311475 428 67.05607477 429 67.13286713 430 67.20930233 431 67.28538283 432 67.36111111 433 67.20554273 434 67.28110599 435 67.35632184 436 67.43119266 437 67.50572082 438 67.57990868 439 67.65375854 440 67.50000000 441 67.57369615 442 67.42081448 443 67.49435666 444 67.56756757 445 67.64044944 446 67.71300448 447 67.78523490 448 67.85714286 449 67.92873051 450 67.77777778 451 67.84922395 452 67.92035398 453 67.99116998 454 67.84140969 455 67.69230769 456 67.76315789 457 67.83369803 458 67.90393013 459 67.97385621 460 67.82608696 461 67.89587852 462 67.96536797 463 67.81857451 464 67.67241379 465 67.74193548 466 67.81115880 467 67.88008565 468 67.94871795 469 67.80383795 470 67.87234043 471 67.94055202 472 68.00847458 473 67.86469345 474 67.93248945 475 68.00000000 476 68.06722689 477 68.13417191 478 68.20083682 479 68.26722338 480 68.33333333 481 68.19126819 482 68.25726141 483 68.32298137 484 68.38842975 485 68.45360825 486 68.51851852 487 68.37782341 488 68.44262295 489 68.30265849 490 68.16326531 491 68.22810591 492 68.29268293 493 68.15415822 494 68.21862348 495 68.08080808 496 68.14516129 497 68.20925553 498 68.27309237 499 68.33667335 500 68.40000000 501 68.26347305 502 68.32669323 503 68.38966203 504 68.45238095 505 68.51485149 506 68.37944664 507 68.24457594 508 68.30708661 509 68.17288802 510 68.23529412 511 68.10176125 512 67.96875000 513 68.03118908 514 68.09338521 515 68.15533981 516 68.02325581 517 68.08510638 518 68.14671815 519 68.20809249 520 68.07692308 521 68.13819578 522 68.19923372 523 68.06883365 524 68.12977099 525 68.00000000 526 67.87072243 527 67.74193548 528 67.61363636 529 67.48582231 530 67.54716981 531 67.60828625 532 67.66917293 533 67.72983114 534 67.60299625 535 67.47663551 536 67.53731343 537 67.59776536 538 67.47211896 539 67.53246753 540 67.40740741 541 67.46765250 542 67.52767528 543 67.58747698 544 67.64705882 545 67.70642202 546 67.58241758 547 67.45886654 548 67.51824818 549 67.57741348 550 67.63636364 551 67.51361162 552 67.39130435 553 67.45027125 554 67.50902527 555 67.56756757 556 67.44604317 557 67.50448833 558 67.38351254 559 67.44186047 560 67.50000000 561 67.55793226 562 67.61565836 563 67.67317940 564 67.55319149 565 67.43362832 566 67.49116608 567 67.54850088 568 67.60563380 569 67.48681898 570 67.36842105 571 67.42556918 572 67.30769231 573 67.36474695 574 67.42160279 575 67.47826087 576 67.53472222 577 67.59098787 578 67.64705882 579 67.70293610 580 67.58620690 581 67.64199656 582 67.69759450 583 67.75300172 584 67.80821918 585 67.86324786 586 67.74744027 587 67.80238501 588 67.85714286 589 67.91171477 590 67.96610169 591 68.02030457 592 68.07432432 593 68.12816189 594 68.18181818 595 68.06722689 596 67.95302013 597 67.83919598 598 67.72575251 599 67.77963272 600 67.83333333 601 67.88685524 602 67.77408638 603 67.66169154 604 67.71523179 605 67.76859504 606 67.82178218 607 67.71004942 608 67.76315789 609 67.65188834 610 67.70491803 611 67.59410802 612 67.64705882 613 67.53670473 614 67.58957655 615 67.64227642 616 67.53246753 617 67.58508914 618 67.47572816 619 67.36672052 620 67.41935484 621 67.47181965 622 67.36334405 623 67.41573034 624 67.46794872 625 67.36000000 626 67.25239617 627 67.30462520 628 67.35668790 629 67.24960254 630 67.30158730 631 67.35340729 632 67.24683544 633 67.14060032 634 67.19242902 635 67.24409449 636 67.29559748 637 67.18995290 638 67.24137931 639 67.29264476 640 67.34375000 641 67.39469579 642 67.28971963 643 67.18506998 644 67.23602484 645 67.13178295 646 67.18266254 647 67.23338485 648 67.12962963 649 67.02619414 650 67.07692308 651 67.12749616 652 67.17791411 653 67.07503828 654 66.97247706 655 66.87022901 656 66.76829268 657 66.66666667 658 66.56534954 659 66.46433991 660 66.51515152 661 66.41452345 662 66.46525680 663 66.51583710 664 66.56626506 665 66.61654135 666 66.66666667 667 66.71664168 668 66.61676647 669 66.51718984 670 66.56716418 671 66.46795827 672 66.51785714 673 66.41901932 674 66.32047478 675 66.37037037 676 66.27218935 677 66.32200886 678 66.37168142 679 66.27393225 680 66.17647059 681 66.22613803 682 66.12903226 683 66.03221083 684 66.08187135 685 66.13138686 686 66.03498542 687 65.93886463 688 65.98837209 689 65.89259797 690 65.94202899 691 65.84659913 692 65.89595376 693 65.94516595 694 65.99423631 695 65.89928058 696 65.80459770 697 65.71018651 698 65.61604585 699 65.52217454 700 65.57142857 701 65.47788873 702 65.52706553 703 65.57610242 704 65.62500000 705 65.67375887 706 65.72237960 707 65.77086280 708 65.81920904 709 65.86741890 710 65.77464789 711 65.82278481 712 65.87078652 713 65.91865358 714 65.82633053 715 65.73426573 716 65.78212291 717 65.82984658 718 65.73816156 719 65.64673157 720 65.69444444 721 65.74202497 722 65.78947368 723 65.83679115 724 65.74585635 725 65.79310345 726 65.70247934 727 65.61210454 728 65.65934066 729 65.70644719 730 65.75342466 731 65.80027360 732 65.84699454 733 65.89358799 734 65.94005450 735 65.98639456 736 66.03260870 737 65.94301221 738 65.98915989 739 66.03518268 740 66.08108108 741 66.12685560 742 66.17250674 743 66.21803499 744 66.12903226 745 66.17449664 746 66.21983914 747 66.13119143 748 66.04278075 749 65.95460614 750 66.00000000 Final result: 66.0000 +/- 1.7309 Random chance: 25.0083 +/- 1.5824