common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 37.50000000 9 33.33333333 10 30.00000000 11 27.27272727 12 25.00000000 13 30.76923077 14 28.57142857 15 33.33333333 16 37.50000000 17 35.29411765 18 33.33333333 19 36.84210526 20 35.00000000 21 33.33333333 22 31.81818182 23 30.43478261 24 29.16666667 25 32.00000000 26 30.76923077 27 29.62962963 28 32.14285714 29 34.48275862 30 36.66666667 31 38.70967742 32 40.62500000 33 39.39393939 34 38.23529412 35 40.00000000 36 38.88888889 37 37.83783784 38 36.84210526 39 35.89743590 40 35.00000000 41 34.14634146 42 33.33333333 43 32.55813953 44 31.81818182 45 31.11111111 46 30.43478261 47 29.78723404 48 31.25000000 49 32.65306122 50 34.00000000 51 33.33333333 52 32.69230769 53 32.07547170 54 31.48148148 55 32.72727273 56 32.14285714 57 33.33333333 58 32.75862069 59 32.20338983 60 31.66666667 61 31.14754098 62 30.64516129 63 30.15873016 64 29.68750000 65 30.76923077 66 31.81818182 67 31.34328358 68 30.88235294 69 30.43478261 70 31.42857143 71 30.98591549 72 31.94444444 73 31.50684932 74 31.08108108 75 32.00000000 76 32.89473684 77 33.76623377 78 34.61538462 79 34.17721519 80 35.00000000 81 35.80246914 82 35.36585366 83 36.14457831 84 36.90476190 85 36.47058824 86 36.04651163 87 36.78160920 88 37.50000000 89 37.07865169 90 36.66666667 91 36.26373626 92 35.86956522 93 35.48387097 94 36.17021277 95 35.78947368 96 36.45833333 97 36.08247423 98 35.71428571 99 35.35353535 100 36.00000000 101 35.64356436 102 35.29411765 103 34.95145631 104 34.61538462 105 34.28571429 106 33.96226415 107 33.64485981 108 33.33333333 109 33.02752294 110 32.72727273 111 32.43243243 112 32.14285714 113 31.85840708 114 32.45614035 115 33.04347826 116 32.75862069 117 32.47863248 118 32.20338983 119 31.93277311 120 31.66666667 121 31.40495868 122 31.14754098 123 30.89430894 124 30.64516129 125 31.20000000 126 30.95238095 127 30.70866142 128 30.46875000 129 30.23255814 130 30.00000000 131 30.53435115 132 31.06060606 133 30.82706767 134 30.59701493 135 30.37037037 136 30.14705882 137 30.65693431 138 30.43478261 139 30.93525180 140 30.71428571 141 31.20567376 142 30.98591549 143 31.46853147 144 31.94444444 145 31.72413793 146 31.50684932 147 31.29251701 148 31.75675676 149 31.54362416 150 31.33333333 151 31.12582781 152 31.57894737 153 31.37254902 154 31.16883117 155 30.96774194 156 30.76923077 157 31.21019108 158 31.01265823 159 30.81761006 160 31.25000000 161 31.05590062 162 30.86419753 163 30.67484663 164 31.09756098 165 31.51515152 166 31.32530120 167 31.73652695 168 31.54761905 169 31.36094675 170 31.17647059 171 30.99415205 172 30.81395349 173 30.63583815 174 30.45977011 175 30.28571429 176 30.11363636 177 29.94350282 178 29.77528090 179 30.16759777 180 30.00000000 181 29.83425414 182 29.67032967 183 30.05464481 184 29.89130435 185 29.72972973 186 29.56989247 187 29.41176471 188 29.25531915 189 29.10052910 190 28.94736842 191 29.31937173 192 29.16666667 193 29.01554404 194 29.38144330 195 29.74358974 196 29.59183673 197 29.44162437 198 29.29292929 199 29.14572864 200 29.00000000 201 29.35323383 202 29.20792079 203 29.06403941 204 28.92156863 205 29.26829268 206 29.12621359 207 28.98550725 208 28.84615385 209 28.70813397 210 28.57142857 211 28.90995261 212 28.77358491 213 28.63849765 214 28.50467290 215 28.37209302 216 28.24074074 217 28.11059908 218 27.98165138 219 27.85388128 220 27.72727273 221 27.60180995 222 27.92792793 223 27.80269058 224 27.67857143 225 28.00000000 226 27.87610619 227 28.19383260 228 28.50877193 229 28.38427948 230 28.26086957 231 28.57142857 232 28.44827586 233 28.32618026 234 28.63247863 235 28.51063830 236 28.38983051 237 28.27004219 238 28.15126050 239 28.03347280 240 28.33333333 241 28.21576763 242 28.51239669 243 28.80658436 244 28.68852459 245 28.97959184 246 28.86178862 247 28.74493927 248 28.62903226 249 28.91566265 250 28.80000000 251 28.68525896 252 28.57142857 253 28.45849802 254 28.34645669 255 28.62745098 256 28.51562500 257 28.40466926 258 28.29457364 259 28.18532819 260 28.07692308 261 27.96934866 262 27.86259542 263 27.75665399 264 27.65151515 265 27.54716981 266 27.81954887 267 27.71535581 268 27.61194030 269 27.50929368 270 27.40740741 271 27.30627306 272 27.57352941 273 27.47252747 274 27.37226277 275 27.63636364 276 27.89855072 277 28.15884477 278 28.41726619 279 28.31541219 280 28.57142857 281 28.46975089 282 28.36879433 283 28.26855124 284 28.16901408 285 28.07017544 286 27.97202797 287 28.22299652 288 28.12500000 289 28.02768166 290 28.27586207 291 28.52233677 292 28.76712329 293 29.01023891 294 29.25170068 295 29.15254237 296 29.05405405 297 28.95622896 298 28.85906040 299 29.09698997 300 29.33333333 301 29.56810631 302 29.80132450 303 29.70297030 304 29.60526316 305 29.83606557 306 29.73856209 307 29.64169381 308 29.54545455 309 29.44983819 310 29.35483871 311 29.26045016 312 29.48717949 313 29.71246006 314 29.93630573 315 29.84126984 316 29.74683544 317 29.65299685 318 29.55974843 319 29.78056426 320 29.68750000 321 29.59501558 322 29.50310559 323 29.41176471 324 29.32098765 325 29.53846154 326 29.44785276 327 29.66360856 328 29.87804878 329 29.78723404 330 29.69696970 331 29.60725076 332 29.81927711 333 29.72972973 334 29.64071856 335 29.85074627 336 29.76190476 337 29.67359050 338 29.58579882 339 29.49852507 340 29.70588235 341 29.61876833 342 29.53216374 343 29.44606414 344 29.36046512 345 29.27536232 346 29.47976879 347 29.68299712 348 29.59770115 349 29.79942693 350 30.00000000 351 29.91452991 352 29.82954545 353 30.02832861 354 29.94350282 355 29.85915493 356 30.05617978 357 29.97198880 358 29.88826816 359 29.80501393 360 30.00000000 361 30.19390582 362 30.38674033 363 30.57851240 364 30.49450549 365 30.41095890 366 30.32786885 367 30.24523161 368 30.16304348 369 30.08130081 370 30.27027027 371 30.45822102 372 30.37634409 373 30.29490617 374 30.21390374 375 30.40000000 376 30.31914894 377 30.23872679 378 30.42328042 379 30.34300792 380 30.26315789 381 30.44619423 382 30.36649215 383 30.54830287 384 30.46875000 385 30.64935065 386 30.56994819 387 30.49095607 388 30.41237113 389 30.59125964 390 30.51282051 391 30.69053708 392 30.86734694 393 30.78880407 394 30.71065990 395 30.63291139 396 30.80808081 397 30.98236776 398 30.90452261 399 31.07769424 400 31.00000000 401 31.17206983 402 31.34328358 403 31.26550868 404 31.18811881 405 31.11111111 406 31.03448276 407 30.95823096 408 30.88235294 409 31.05134474 410 30.97560976 411 30.90024331 412 30.82524272 413 30.99273608 414 31.15942029 415 31.32530120 416 31.25000000 417 31.17505995 418 31.10047847 419 31.26491647 420 31.42857143 421 31.35391924 422 31.27962085 423 31.20567376 424 31.36792453 425 31.52941176 426 31.45539906 427 31.38173302 428 31.30841121 429 31.46853147 430 31.39534884 431 31.55452436 432 31.71296296 433 31.63972286 434 31.56682028 435 31.49425287 436 31.42201835 437 31.35011442 438 31.50684932 439 31.66287016 440 31.59090909 441 31.51927438 442 31.44796380 443 31.37697517 444 31.53153153 445 31.46067416 446 31.39013453 447 31.31991051 448 31.25000000 449 31.18040089 450 31.11111111 451 31.04212860 452 31.19469027 453 31.34657837 454 31.49779736 455 31.42857143 456 31.57894737 457 31.50984683 458 31.44104803 459 31.59041394 460 31.73913043 461 31.67028200 462 31.60173160 463 31.53347732 464 31.46551724 465 31.39784946 466 31.33047210 467 31.47751606 468 31.41025641 469 31.55650320 470 31.48936170 471 31.42250531 472 31.35593220 473 31.50105708 474 31.43459916 475 31.36842105 476 31.51260504 477 31.44654088 478 31.58995816 479 31.73277662 480 31.66666667 481 31.80873181 482 31.74273859 483 31.67701863 484 31.61157025 485 31.54639175 486 31.48148148 487 31.62217659 488 31.55737705 489 31.49284254 490 31.63265306 491 31.77189409 492 31.70731707 493 31.64300203 494 31.78137652 495 31.91919192 496 32.05645161 497 31.99195171 498 31.92771084 499 31.86372745 500 32.00000000 501 31.93612774 502 31.87250996 503 31.80914513 504 31.94444444 505 31.88118812 506 32.01581028 507 31.95266272 508 31.88976378 509 31.82711198 510 31.96078431 511 31.89823875 512 31.83593750 513 31.77387914 514 31.71206226 515 31.65048544 516 31.58914729 517 31.72147002 518 31.85328185 519 31.79190751 520 31.73076923 521 31.66986564 522 31.60919540 523 31.73996176 524 31.87022901 525 31.80952381 526 31.74904943 527 31.87855787 528 31.81818182 529 31.94706994 530 32.07547170 531 32.01506591 532 31.95488722 533 31.89493433 534 31.83520599 535 31.96261682 536 31.90298507 537 32.02979516 538 32.15613383 539 32.09647495 540 32.03703704 541 31.97781885 542 32.10332103 543 32.04419890 544 31.98529412 545 31.92660550 546 31.86813187 547 31.80987203 548 31.93430657 549 31.87613843 550 31.81818182 551 31.94192377 552 31.88405797 553 31.82640145 554 31.76895307 555 31.71171171 556 31.83453237 557 31.77737882 558 31.89964158 559 31.84257603 560 31.78571429 561 31.72905526 562 31.67259786 563 31.79396092 564 31.73758865 565 31.85840708 566 31.80212014 567 31.74603175 568 31.86619718 569 31.81019332 570 31.75438596 571 31.69877408 572 31.64335664 573 31.58813264 574 31.70731707 575 31.65217391 576 31.77083333 577 31.88908146 578 31.83391003 579 31.95164076 580 31.89655172 581 31.84165232 582 31.78694158 583 31.90394511 584 32.02054795 585 31.96581197 586 32.08191126 587 32.02725724 588 31.97278912 589 31.91850594 590 31.86440678 591 31.81049069 592 31.75675676 593 31.70320405 594 31.81818182 595 31.76470588 596 31.87919463 597 31.82579564 598 31.77257525 599 31.88647746 600 31.83333333 601 31.94675541 602 31.89368771 603 32.00663350 604 31.95364238 605 32.06611570 606 32.01320132 607 32.12520593 608 32.07236842 609 32.01970443 610 31.96721311 611 31.91489362 612 31.86274510 613 31.97389886 614 32.08469055 615 32.03252033 616 32.14285714 617 32.09076175 618 32.03883495 619 31.98707593 620 31.93548387 621 31.88405797 622 31.83279743 623 31.78170144 624 31.73076923 625 31.68000000 626 31.78913738 627 31.73843700 628 31.68789809 629 31.79650238 630 31.90476190 631 31.85419968 632 31.96202532 633 31.91153239 634 31.86119874 635 31.81102362 636 31.76100629 637 31.71114600 638 31.66144201 639 31.76838811 640 31.87500000 641 31.82527301 642 31.93146417 643 31.88180404 644 31.98757764 645 31.93798450 646 31.88854489 647 31.83925811 648 31.79012346 649 31.74114022 650 31.69230769 651 31.64362519 652 31.74846626 653 31.69984686 654 31.80428135 655 31.75572519 656 31.85975610 657 31.81126332 658 31.91489362 659 31.86646434 660 31.81818182 661 31.77004539 662 31.72205438 663 31.67420814 664 31.77710843 665 31.72932331 666 31.68168168 667 31.63418291 668 31.73652695 669 31.68908819 670 31.64179104 671 31.59463487 672 31.54761905 673 31.50074294 674 31.45400593 675 31.40740741 676 31.36094675 677 31.31462334 678 31.41592920 679 31.36966127 680 31.47058824 681 31.57121880 682 31.52492669 683 31.62518302 684 31.57894737 685 31.53284672 686 31.48688047 687 31.44104803 688 31.39534884 689 31.34978229 690 31.30434783 691 31.25904486 692 31.21387283 693 31.16883117 694 31.26801153 695 31.22302158 696 31.32183908 697 31.27690100 698 31.23209169 699 31.33047210 700 31.28571429 701 31.38373752 702 31.48148148 703 31.57894737 704 31.53409091 705 31.48936170 706 31.44475921 707 31.40028289 708 31.35593220 709 31.31170663 710 31.40845070 711 31.50492264 712 31.46067416 713 31.55680224 714 31.65266106 715 31.60839161 716 31.56424581 717 31.52022315 718 31.47632312 719 31.43254520 720 31.38888889 721 31.34535368 722 31.30193906 723 31.39695712 724 31.35359116 725 31.44827586 726 31.54269972 727 31.49931224 728 31.45604396 729 31.41289438 730 31.36986301 731 31.32694938 732 31.42076503 733 31.51432469 734 31.60762943 735 31.56462585 736 31.52173913 737 31.61465400 738 31.57181572 739 31.66441137 740 31.62162162 741 31.57894737 742 31.67115903 743 31.76312248 744 31.85483871 745 31.81208054 746 31.76943700 747 31.72690763 748 31.68449198 749 31.64218959 750 31.73333333 Final result: 31.7333 ±1.7007 Random chance: 25.0000 ±1.5822