common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 37.50000000 9 33.33333333 10 30.00000000 11 27.27272727 12 25.00000000 13 30.76923077 14 28.57142857 15 33.33333333 16 37.50000000 17 35.29411765 18 33.33333333 19 36.84210526 20 35.00000000 21 33.33333333 22 31.81818182 23 30.43478261 24 29.16666667 25 32.00000000 26 30.76923077 27 29.62962963 28 32.14285714 29 34.48275862 30 36.66666667 31 38.70967742 32 40.62500000 33 39.39393939 34 38.23529412 35 40.00000000 36 38.88888889 37 37.83783784 38 36.84210526 39 35.89743590 40 35.00000000 41 34.14634146 42 33.33333333 43 32.55813953 44 31.81818182 45 31.11111111 46 30.43478261 47 29.78723404 48 31.25000000 49 32.65306122 50 34.00000000 51 33.33333333 52 32.69230769 53 32.07547170 54 31.48148148 55 32.72727273 56 33.92857143 57 35.08771930 58 34.48275862 59 33.89830508 60 33.33333333 61 32.78688525 62 32.25806452 63 31.74603175 64 31.25000000 65 32.30769231 66 33.33333333 67 32.83582090 68 32.35294118 69 31.88405797 70 32.85714286 71 32.39436620 72 33.33333333 73 32.87671233 74 32.43243243 75 33.33333333 76 34.21052632 77 35.06493506 78 35.89743590 79 35.44303797 80 36.25000000 81 37.03703704 82 36.58536585 83 37.34939759 84 38.09523810 85 37.64705882 86 37.20930233 87 37.93103448 88 38.63636364 89 38.20224719 90 37.77777778 91 37.36263736 92 36.95652174 93 36.55913978 94 36.17021277 95 35.78947368 96 36.45833333 97 36.08247423 98 35.71428571 99 35.35353535 100 36.00000000 101 35.64356436 102 35.29411765 103 34.95145631 104 34.61538462 105 35.23809524 106 34.90566038 107 34.57943925 108 34.25925926 109 33.94495413 110 33.63636364 111 33.33333333 112 33.03571429 113 32.74336283 114 33.33333333 115 33.91304348 116 33.62068966 117 33.33333333 118 33.05084746 119 32.77310924 120 32.50000000 121 32.23140496 122 31.96721311 123 31.70731707 124 31.45161290 125 32.00000000 126 31.74603175 127 31.49606299 128 31.25000000 129 31.00775194 130 30.76923077 131 31.29770992 132 31.81818182 133 31.57894737 134 31.34328358 135 31.11111111 136 30.88235294 137 30.65693431 138 30.43478261 139 30.21582734 140 30.00000000 141 30.49645390 142 30.28169014 143 30.76923077 144 31.25000000 145 31.03448276 146 30.82191781 147 30.61224490 148 31.08108108 149 30.87248322 150 30.66666667 151 30.46357616 152 30.26315789 153 30.06535948 154 29.87012987 155 29.67741935 156 29.48717949 157 29.93630573 158 29.74683544 159 29.55974843 160 30.00000000 161 29.81366460 162 29.62962963 163 29.44785276 164 29.26829268 165 29.69696970 166 29.51807229 167 29.94011976 168 29.76190476 169 29.58579882 170 29.41176471 171 29.23976608 172 29.06976744 173 28.90173410 174 28.73563218 175 28.57142857 176 28.40909091 177 28.24858757 178 28.08988764 179 28.49162011 180 28.33333333 181 28.17679558 182 28.02197802 183 27.86885246 184 27.71739130 185 27.56756757 186 27.95698925 187 27.80748663 188 27.65957447 189 27.51322751 190 27.36842105 191 27.74869110 192 27.60416667 193 27.46113990 194 27.83505155 195 28.20512821 196 28.06122449 197 27.91878173 198 27.77777778 199 27.63819095 200 27.50000000 201 27.86069652 202 27.72277228 203 27.58620690 204 27.45098039 205 27.80487805 206 27.66990291 207 27.53623188 208 27.40384615 209 27.27272727 210 27.14285714 211 27.48815166 212 27.35849057 213 27.23004695 214 27.10280374 215 26.97674419 216 27.31481481 217 27.18894009 218 27.06422018 219 26.94063927 220 26.81818182 221 26.69683258 222 27.02702703 223 26.90582960 224 26.78571429 225 27.11111111 226 26.99115044 227 27.31277533 228 27.63157895 229 27.51091703 230 27.39130435 231 27.27272727 232 27.58620690 233 27.46781116 234 27.77777778 235 27.65957447 236 27.54237288 237 27.42616034 238 27.31092437 239 27.19665272 240 27.50000000 241 27.38589212 242 27.68595041 243 27.98353909 244 27.86885246 245 28.16326531 246 28.04878049 247 27.93522267 248 28.22580645 249 28.11244980 250 28.00000000 251 27.88844622 252 27.77777778 253 27.66798419 254 27.55905512 255 27.84313725 256 27.73437500 257 28.01556420 258 27.90697674 259 27.79922780 260 27.69230769 261 27.58620690 262 27.48091603 263 27.37642586 264 27.27272727 265 27.16981132 266 27.06766917 267 26.96629213 268 26.86567164 269 26.76579926 270 26.66666667 271 26.56826568 272 26.83823529 273 26.73992674 274 26.64233577 275 26.54545455 276 26.81159420 277 27.07581227 278 27.33812950 279 27.24014337 280 27.50000000 281 27.40213523 282 27.30496454 283 27.20848057 284 27.11267606 285 27.01754386 286 26.92307692 287 27.17770035 288 27.08333333 289 26.98961938 290 27.24137931 291 27.49140893 292 27.73972603 293 27.98634812 294 28.23129252 295 28.13559322 296 28.04054054 297 27.94612795 298 28.18791946 299 28.42809365 300 28.66666667 301 28.90365449 302 29.13907285 303 29.04290429 304 29.27631579 305 29.50819672 306 29.41176471 307 29.31596091 308 29.22077922 309 29.12621359 310 29.03225806 311 29.26045016 312 29.48717949 313 29.71246006 314 29.93630573 315 29.84126984 316 29.74683544 317 29.65299685 318 29.55974843 319 29.78056426 320 29.68750000 321 29.59501558 322 29.50310559 323 29.41176471 324 29.32098765 325 29.53846154 326 29.44785276 327 29.66360856 328 29.87804878 329 29.78723404 330 29.69696970 331 29.60725076 332 29.81927711 333 29.72972973 334 29.64071856 335 29.85074627 336 29.76190476 337 29.67359050 338 29.58579882 339 29.49852507 340 29.70588235 341 29.91202346 342 29.82456140 343 30.02915452 344 29.94186047 345 29.85507246 346 30.05780347 347 30.25936599 348 30.17241379 349 30.37249284 350 30.57142857 351 30.48433048 352 30.39772727 353 30.59490085 354 30.50847458 355 30.42253521 356 30.61797753 357 30.53221289 358 30.44692737 359 30.36211699 360 30.55555556 361 30.74792244 362 30.93922652 363 31.12947658 364 31.04395604 365 30.95890411 366 30.87431694 367 30.79019074 368 30.70652174 369 30.62330623 370 30.81081081 371 30.72776280 372 30.64516129 373 30.56300268 374 30.74866310 375 30.93333333 376 30.85106383 377 30.76923077 378 30.68783069 379 30.60686016 380 30.52631579 381 30.70866142 382 30.62827225 383 30.80939948 384 30.72916667 385 30.90909091 386 30.82901554 387 30.74935401 388 30.67010309 389 30.84832905 390 30.76923077 391 30.94629156 392 31.12244898 393 31.04325700 394 30.96446701 395 30.88607595 396 31.06060606 397 30.98236776 398 30.90452261 399 31.07769424 400 31.00000000 401 31.17206983 402 31.34328358 403 31.26550868 404 31.18811881 405 31.11111111 406 31.03448276 407 30.95823096 408 30.88235294 409 31.05134474 410 30.97560976 411 30.90024331 412 30.82524272 413 30.99273608 414 31.15942029 415 31.32530120 416 31.49038462 417 31.41486811 418 31.33971292 419 31.50357995 420 31.66666667 421 31.59144893 422 31.51658768 423 31.44208038 424 31.60377358 425 31.76470588 426 31.92488263 427 31.85011710 428 31.77570093 429 31.93473193 430 31.86046512 431 32.01856148 432 32.17592593 433 32.10161663 434 32.25806452 435 32.18390805 436 32.11009174 437 32.03661327 438 32.19178082 439 32.34624146 440 32.27272727 441 32.19954649 442 32.12669683 443 32.05417607 444 31.98198198 445 31.91011236 446 31.83856502 447 31.76733781 448 31.69642857 449 31.62583519 450 31.55555556 451 31.48558758 452 31.63716814 453 31.78807947 454 31.71806167 455 31.64835165 456 31.79824561 457 31.94748359 458 31.87772926 459 32.02614379 460 32.17391304 461 32.10412148 462 32.03463203 463 32.18142549 464 32.32758621 465 32.25806452 466 32.18884120 467 32.33404711 468 32.26495726 469 32.40938166 470 32.34042553 471 32.27176221 472 32.20338983 473 32.34672304 474 32.27848101 475 32.21052632 476 32.35294118 477 32.28511530 478 32.42677824 479 32.56784969 480 32.50000000 481 32.64033264 482 32.57261411 483 32.50517598 484 32.43801653 485 32.37113402 486 32.30452675 487 32.44353183 488 32.37704918 489 32.31083845 490 32.44897959 491 32.58655804 492 32.52032520 493 32.45436105 494 32.59109312 495 32.72727273 496 32.86290323 497 32.79678068 498 32.73092369 499 32.66533066 500 32.80000000 501 32.73453094 502 32.66932271 503 32.60437376 504 32.73809524 505 32.67326733 506 32.80632411 507 32.74161736 508 32.67716535 509 32.61296660 510 32.54901961 511 32.48532290 512 32.42187500 513 32.35867446 514 32.29571984 515 32.23300971 516 32.17054264 517 32.30174081 518 32.43243243 519 32.36994220 520 32.30769231 521 32.24568138 522 32.18390805 523 32.31357553 524 32.44274809 525 32.38095238 526 32.31939163 527 32.44781784 528 32.38636364 529 32.51417769 530 32.64150943 531 32.58003766 532 32.51879699 533 32.64540338 534 32.58426966 535 32.71028037 536 32.64925373 537 32.77467412 538 32.89962825 539 32.83858998 540 32.77777778 541 32.71719039 542 32.84132841 543 32.78084715 544 32.72058824 545 32.66055046 546 32.60073260 547 32.54113346 548 32.66423358 549 32.60473588 550 32.54545455 551 32.66787659 552 32.60869565 553 32.54972875 554 32.49097473 555 32.43243243 556 32.37410072 557 32.31597846 558 32.43727599 559 32.37924866 560 32.32142857 561 32.26381462 562 32.20640569 563 32.32682060 564 32.26950355 565 32.38938053 566 32.33215548 567 32.27513228 568 32.39436620 569 32.33743409 570 32.28070175 571 32.39929947 572 32.34265734 573 32.28621291 574 32.40418118 575 32.34782609 576 32.46527778 577 32.58232236 578 32.52595156 579 32.46977547 580 32.41379310 581 32.35800344 582 32.30240550 583 32.41852487 584 32.53424658 585 32.47863248 586 32.42320819 587 32.36797274 588 32.31292517 589 32.25806452 590 32.20338983 591 32.14890017 592 32.09459459 593 32.04047218 594 32.15488215 595 32.10084034 596 32.21476510 597 32.16080402 598 32.10702341 599 32.22036728 600 32.16666667 601 32.27953411 602 32.22591362 603 32.33830846 604 32.28476821 605 32.23140496 606 32.17821782 607 32.28995058 608 32.23684211 609 32.18390805 610 32.13114754 611 32.07855974 612 32.02614379 613 32.13703100 614 32.24755700 615 32.19512195 616 32.30519481 617 32.25283630 618 32.20064725 619 32.14862682 620 32.09677419 621 32.04508857 622 31.99356913 623 31.94221509 624 31.89102564 625 31.84000000 626 31.94888179 627 31.89792663 628 31.84713376 629 31.95548490 630 32.06349206 631 32.01267829 632 32.12025316 633 32.06951027 634 32.01892744 635 31.96850394 636 31.91823899 637 31.86813187 638 31.81818182 639 31.76838811 640 31.87500000 641 31.82527301 642 31.93146417 643 31.88180404 644 31.98757764 645 31.93798450 646 31.88854489 647 31.83925811 648 31.79012346 649 31.74114022 650 31.69230769 651 31.64362519 652 31.74846626 653 31.69984686 654 31.80428135 655 31.75572519 656 31.85975610 657 31.81126332 658 31.91489362 659 31.86646434 660 31.81818182 661 31.77004539 662 31.72205438 663 31.67420814 664 31.77710843 665 31.72932331 666 31.68168168 667 31.78410795 668 31.88622754 669 31.83856502 670 31.79104478 671 31.74366617 672 31.69642857 673 31.64933135 674 31.60237389 675 31.55555556 676 31.50887574 677 31.46233383 678 31.56342183 679 31.51693667 680 31.61764706 681 31.57121880 682 31.52492669 683 31.62518302 684 31.57894737 685 31.53284672 686 31.48688047 687 31.44104803 688 31.39534884 689 31.34978229 690 31.30434783 691 31.40376266 692 31.35838150 693 31.31313131 694 31.41210375 695 31.51079137 696 31.60919540 697 31.56384505 698 31.51862464 699 31.61659514 700 31.57142857 701 31.66904422 702 31.76638177 703 31.86344239 704 31.81818182 705 31.77304965 706 31.72804533 707 31.68316832 708 31.63841808 709 31.59379408 710 31.69014085 711 31.78621660 712 31.74157303 713 31.83730715 714 31.93277311 715 31.88811189 716 31.98324022 717 31.93863319 718 31.89415042 719 31.84979138 720 31.80555556 721 31.76144244 722 31.71745152 723 31.81189488 724 31.76795580 725 31.86206897 726 31.95592287 727 31.91196699 728 31.86813187 729 31.82441701 730 31.78082192 731 31.73734610 732 31.83060109 733 31.92360164 734 32.01634877 735 31.97278912 736 31.92934783 737 32.02170963 738 31.97831978 739 32.07036536 740 32.16216216 741 32.11875843 742 32.21024259 743 32.30148048 744 32.39247312 745 32.34899329 746 32.30563003 747 32.26238286 748 32.35294118 749 32.30974633 750 32.40000000 Final result: 32.4000 ±1.7100 Random chance: 25.0000 ±1.5822