common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 817 tasks in prompt multiple_choice_score: selecting 750 random tasks from 817 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 25.00000000 5 20.00000000 6 16.66666667 7 14.28571429 8 12.50000000 9 11.11111111 10 10.00000000 11 9.09090909 12 16.66666667 13 15.38461538 14 14.28571429 15 13.33333333 16 18.75000000 17 23.52941176 18 22.22222222 19 26.31578947 20 25.00000000 21 23.80952381 22 22.72727273 23 21.73913043 24 25.00000000 25 24.00000000 26 26.92307692 27 25.92592593 28 25.00000000 29 27.58620690 30 26.66666667 31 29.03225806 32 28.12500000 33 30.30303030 34 29.41176471 35 28.57142857 36 27.77777778 37 29.72972973 38 28.94736842 39 28.20512821 40 27.50000000 41 26.82926829 42 26.19047619 43 27.90697674 44 29.54545455 45 28.88888889 46 30.43478261 47 29.78723404 48 31.25000000 49 30.61224490 50 32.00000000 51 31.37254902 52 30.76923077 53 32.07547170 54 33.33333333 55 34.54545455 56 33.92857143 57 33.33333333 58 32.75862069 59 32.20338983 60 33.33333333 61 32.78688525 62 33.87096774 63 33.33333333 64 32.81250000 65 32.30769231 66 31.81818182 67 31.34328358 68 30.88235294 69 31.88405797 70 31.42857143 71 32.39436620 72 33.33333333 73 34.24657534 74 33.78378378 75 33.33333333 76 32.89473684 77 32.46753247 78 32.05128205 79 31.64556962 80 31.25000000 81 30.86419753 82 31.70731707 83 31.32530120 84 30.95238095 85 30.58823529 86 30.23255814 87 29.88505747 88 29.54545455 89 30.33707865 90 30.00000000 91 30.76923077 92 30.43478261 93 30.10752688 94 29.78723404 95 30.52631579 96 31.25000000 97 30.92783505 98 30.61224490 99 31.31313131 100 31.00000000 101 31.68316832 102 31.37254902 103 32.03883495 104 31.73076923 105 31.42857143 106 31.13207547 107 31.77570093 108 32.40740741 109 32.11009174 110 32.72727273 111 32.43243243 112 32.14285714 113 31.85840708 114 31.57894737 115 31.30434783 116 31.03448276 117 30.76923077 118 30.50847458 119 30.25210084 120 30.00000000 121 30.57851240 122 30.32786885 123 30.08130081 124 30.64516129 125 31.20000000 126 31.74603175 127 31.49606299 128 31.25000000 129 31.00775194 130 30.76923077 131 30.53435115 132 30.30303030 133 30.07518797 134 30.59701493 135 31.11111111 136 30.88235294 137 31.38686131 138 31.15942029 139 31.65467626 140 31.42857143 141 31.20567376 142 30.98591549 143 30.76923077 144 30.55555556 145 30.34482759 146 30.13698630 147 29.93197279 148 29.72972973 149 29.53020134 150 30.00000000 151 30.46357616 152 30.26315789 153 30.06535948 154 29.87012987 155 29.67741935 156 30.12820513 157 29.93630573 158 30.37974684 159 30.18867925 160 30.00000000 161 29.81366460 162 29.62962963 163 29.44785276 164 29.26829268 165 29.69696970 166 30.12048193 167 29.94011976 168 29.76190476 169 29.58579882 170 29.41176471 171 29.23976608 172 29.06976744 173 28.90173410 174 28.73563218 175 28.57142857 176 28.97727273 177 28.81355932 178 28.65168539 179 28.49162011 180 28.33333333 181 28.17679558 182 28.02197802 183 27.86885246 184 27.71739130 185 27.56756757 186 27.95698925 187 27.80748663 188 27.65957447 189 28.04232804 190 27.89473684 191 27.74869110 192 27.60416667 193 27.46113990 194 27.83505155 195 27.69230769 196 27.55102041 197 27.91878173 198 28.28282828 199 28.14070352 200 28.00000000 201 27.86069652 202 28.21782178 203 28.07881773 204 27.94117647 205 28.29268293 206 28.15533981 207 28.50241546 208 28.36538462 209 28.22966507 210 28.09523810 211 27.96208531 212 27.83018868 213 27.69953052 214 27.57009346 215 27.44186047 216 27.77777778 217 27.64976959 218 27.98165138 219 27.85388128 220 27.72727273 221 27.60180995 222 27.47747748 223 27.35426009 224 27.67857143 225 27.55555556 226 27.43362832 227 27.31277533 228 27.19298246 229 27.07423581 230 26.95652174 231 26.83982684 232 27.15517241 233 27.46781116 234 27.77777778 235 27.65957447 236 27.96610169 237 28.27004219 238 28.57142857 239 28.45188285 240 28.33333333 241 28.21576763 242 28.09917355 243 27.98353909 244 27.86885246 245 27.75510204 246 27.64227642 247 27.53036437 248 27.41935484 249 27.71084337 250 27.60000000 251 27.88844622 252 27.77777778 253 27.66798419 254 27.55905512 255 27.45098039 256 27.34375000 257 27.23735409 258 27.13178295 259 27.02702703 260 26.92307692 261 27.20306513 262 27.09923664 263 26.99619772 264 27.27272727 265 27.54716981 266 27.44360902 267 27.71535581 268 27.61194030 269 27.50929368 270 27.40740741 271 27.30627306 272 27.20588235 273 27.10622711 274 27.00729927 275 26.90909091 276 26.81159420 277 26.71480144 278 26.61870504 279 26.52329749 280 26.78571429 281 27.04626335 282 26.95035461 283 26.85512367 284 26.76056338 285 26.66666667 286 26.92307692 287 27.17770035 288 27.08333333 289 26.98961938 290 26.89655172 291 27.14776632 292 27.39726027 293 27.64505119 294 27.55102041 295 27.79661017 296 28.04054054 297 27.94612795 298 27.85234899 299 27.75919732 300 27.66666667 301 27.57475083 302 27.48344371 303 27.39273927 304 27.30263158 305 27.21311475 306 27.12418301 307 27.03583062 308 27.27272727 309 27.18446602 310 27.09677419 311 27.00964630 312 27.24358974 313 27.15654952 314 27.07006369 315 26.98412698 316 26.89873418 317 26.81388013 318 26.72955975 319 26.64576803 320 26.56250000 321 26.47975078 322 26.70807453 323 26.62538700 324 26.54320988 325 26.46153846 326 26.38036810 327 26.29969419 328 26.52439024 329 26.44376900 330 26.66666667 331 26.58610272 332 26.80722892 333 26.72672673 334 26.64670659 335 26.86567164 336 26.78571429 337 26.70623145 338 26.62721893 339 26.84365782 340 26.76470588 341 26.68621701 342 26.60818713 343 26.53061224 344 26.74418605 345 26.95652174 346 27.16763006 347 27.37752161 348 27.29885057 349 27.22063037 350 27.14285714 351 27.06552707 352 27.27272727 353 27.47875354 354 27.40112994 355 27.60563380 356 27.52808989 357 27.45098039 358 27.65363128 359 27.57660167 360 27.50000000 361 27.42382271 362 27.34806630 363 27.54820937 364 27.47252747 365 27.67123288 366 27.59562842 367 27.79291553 368 27.71739130 369 27.64227642 370 27.56756757 371 27.49326146 372 27.68817204 373 27.61394102 374 27.80748663 375 27.73333333 376 27.65957447 377 27.58620690 378 27.51322751 379 27.44063325 380 27.36842105 381 27.29658793 382 27.22513089 383 27.15404700 384 27.08333333 385 27.01298701 386 26.94300518 387 26.87338501 388 27.06185567 389 26.99228792 390 26.92307692 391 27.10997442 392 27.04081633 393 26.97201018 394 27.15736041 395 27.08860759 396 27.02020202 397 27.20403023 398 27.13567839 399 27.06766917 400 27.00000000 401 26.93266833 402 27.11442786 403 27.29528536 404 27.22772277 405 27.16049383 406 27.09359606 407 27.02702703 408 26.96078431 409 26.89486553 410 26.82926829 411 26.76399027 412 26.94174757 413 26.87651332 414 27.05314010 415 26.98795181 416 26.92307692 417 26.85851319 418 26.79425837 419 26.73031026 420 26.90476190 421 27.07838480 422 27.01421801 423 26.95035461 424 26.88679245 425 26.82352941 426 26.76056338 427 26.69789227 428 26.63551402 429 26.57342657 430 26.51162791 431 26.45011601 432 26.62037037 433 26.78983834 434 26.95852535 435 26.89655172 436 27.06422018 437 27.00228833 438 26.94063927 439 26.87927107 440 26.81818182 441 26.75736961 442 26.92307692 443 26.86230248 444 26.80180180 445 26.74157303 446 26.68161435 447 26.62192394 448 26.56250000 449 26.50334076 450 26.44444444 451 26.38580931 452 26.32743363 453 26.49006623 454 26.43171806 455 26.37362637 456 26.31578947 457 26.25820569 458 26.20087336 459 26.14379085 460 26.08695652 461 26.03036876 462 25.97402597 463 25.91792657 464 25.86206897 465 26.02150538 466 25.96566524 467 25.91006424 468 25.85470085 469 25.79957356 470 25.74468085 471 25.69002123 472 25.63559322 473 25.58139535 474 25.52742616 475 25.68421053 476 25.63025210 477 25.78616352 478 25.73221757 479 25.67849687 480 25.62500000 481 25.57172557 482 25.72614108 483 25.67287785 484 25.82644628 485 25.97938144 486 25.92592593 487 26.07802875 488 26.02459016 489 26.17586912 490 26.32653061 491 26.27291242 492 26.42276423 493 26.36916836 494 26.31578947 495 26.46464646 496 26.41129032 497 26.35814889 498 26.30522088 499 26.25250501 500 26.20000000 501 26.34730539 502 26.29482072 503 26.24254473 504 26.38888889 505 26.53465347 506 26.48221344 507 26.42998028 508 26.37795276 509 26.52259332 510 26.66666667 511 26.61448141 512 26.56250000 513 26.51072125 514 26.45914397 515 26.60194175 516 26.55038760 517 26.69245648 518 26.64092664 519 26.58959538 520 26.53846154 521 26.67946257 522 26.62835249 523 26.76864245 524 26.71755725 525 26.85714286 526 26.80608365 527 26.94497154 528 26.89393939 529 26.84310019 530 26.79245283 531 26.93032015 532 26.87969925 533 27.01688555 534 26.96629213 535 26.91588785 536 26.86567164 537 27.00186220 538 26.95167286 539 26.90166976 540 27.03703704 541 26.98706100 542 26.93726937 543 26.88766114 544 26.83823529 545 26.78899083 546 26.92307692 547 26.87385740 548 27.00729927 549 26.95810565 550 26.90909091 551 26.86025408 552 26.99275362 553 26.94394213 554 26.89530686 555 26.84684685 556 26.97841727 557 26.92998205 558 26.88172043 559 26.83363148 560 26.78571429 561 26.73796791 562 26.69039146 563 26.64298401 564 26.77304965 565 26.90265487 566 26.85512367 567 26.80776014 568 26.93661972 569 27.06502636 570 27.01754386 571 26.97022767 572 26.92307692 573 26.87609075 574 27.00348432 575 27.13043478 576 27.08333333 577 27.03639515 578 27.16262976 579 27.28842832 580 27.41379310 581 27.36660929 582 27.49140893 583 27.44425386 584 27.56849315 585 27.52136752 586 27.64505119 587 27.59795571 588 27.55102041 589 27.50424448 590 27.62711864 591 27.58037225 592 27.53378378 593 27.48735245 594 27.44107744 595 27.39495798 596 27.34899329 597 27.30318258 598 27.25752508 599 27.21202003 600 27.16666667 601 27.28785358 602 27.40863787 603 27.36318408 604 27.31788079 605 27.27272727 606 27.39273927 607 27.34761120 608 27.30263158 609 27.25779967 610 27.21311475 611 27.16857610 612 27.28758170 613 27.24306688 614 27.19869707 615 27.15447154 616 27.27272727 617 27.39059968 618 27.34627832 619 27.30210016 620 27.25806452 621 27.21417069 622 27.33118971 623 27.28731942 624 27.24358974 625 27.20000000 626 27.15654952 627 27.11323764 628 27.07006369 629 27.02702703 630 26.98412698 631 27.09984152 632 27.21518987 633 27.17219589 634 27.28706625 635 27.24409449 636 27.20125786 637 27.31554160 638 27.27272727 639 27.23004695 640 27.18750000 641 27.14508580 642 27.25856698 643 27.21617418 644 27.32919255 645 27.44186047 646 27.39938080 647 27.35703246 648 27.46913580 649 27.42681048 650 27.53846154 651 27.64976959 652 27.76073620 653 27.87136294 654 27.82874618 655 27.78625954 656 27.74390244 657 27.85388128 658 27.81155015 659 27.76934750 660 27.72727273 661 27.68532526 662 27.64350453 663 27.60180995 664 27.56024096 665 27.66917293 666 27.77777778 667 27.73613193 668 27.69461078 669 27.65321375 670 27.61194030 671 27.57078987 672 27.52976190 673 27.48885587 674 27.44807122 675 27.55555556 676 27.66272189 677 27.62186115 678 27.58112094 679 27.54050074 680 27.64705882 681 27.60646109 682 27.56598240 683 27.52562225 684 27.63157895 685 27.59124088 686 27.55102041 687 27.51091703 688 27.61627907 689 27.72133527 690 27.68115942 691 27.78581766 692 27.74566474 693 27.70562771 694 27.80979827 695 27.76978417 696 27.87356322 697 27.83357245 698 27.79369628 699 27.89699571 700 28.00000000 701 27.96005706 702 27.92022792 703 27.88051209 704 27.84090909 705 27.80141844 706 27.76203966 707 27.86421499 708 27.82485876 709 27.78561354 710 27.74647887 711 27.70745429 712 27.80898876 713 27.76998597 714 27.73109244 715 27.69230769 716 27.65363128 717 27.61506276 718 27.57660167 719 27.53824757 720 27.50000000 721 27.46185853 722 27.42382271 723 27.38589212 724 27.34806630 725 27.31034483 726 27.41046832 727 27.37276479 728 27.33516484 729 27.29766804 730 27.39726027 731 27.35978112 732 27.45901639 733 27.42155525 734 27.38419619 735 27.34693878 736 27.30978261 737 27.27272727 738 27.23577236 739 27.33423545 740 27.29729730 741 27.26045884 742 27.22371968 743 27.18707941 744 27.28494624 745 27.24832215 746 27.34584450 747 27.44310576 748 27.40641711 749 27.50333778 750 27.60000000 Final result: 27.6000 ±1.6334 Random chance: 19.8992 ±1.4588