common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 817 tasks in prompt multiple_choice_score: selecting 750 random tasks from 817 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 0.00000000 2 0.00000000 3 0.00000000 4 0.00000000 5 20.00000000 6 16.66666667 7 14.28571429 8 25.00000000 9 22.22222222 10 20.00000000 11 18.18181818 12 25.00000000 13 30.76923077 14 28.57142857 15 33.33333333 16 37.50000000 17 35.29411765 18 33.33333333 19 31.57894737 20 30.00000000 21 28.57142857 22 27.27272727 23 26.08695652 24 29.16666667 25 28.00000000 26 26.92307692 27 25.92592593 28 25.00000000 29 27.58620690 30 30.00000000 31 32.25806452 32 34.37500000 33 36.36363636 34 35.29411765 35 34.28571429 36 33.33333333 37 35.13513514 38 34.21052632 39 35.89743590 40 35.00000000 41 34.14634146 42 33.33333333 43 34.88372093 44 36.36363636 45 37.77777778 46 36.95652174 47 36.17021277 48 37.50000000 49 36.73469388 50 38.00000000 51 37.25490196 52 36.53846154 53 35.84905660 54 37.03703704 55 38.18181818 56 37.50000000 57 36.84210526 58 36.20689655 59 37.28813559 60 38.33333333 61 37.70491803 62 38.70967742 63 38.09523810 64 37.50000000 65 36.92307692 66 36.36363636 67 35.82089552 68 35.29411765 69 36.23188406 70 35.71428571 71 35.21126761 72 36.11111111 73 36.98630137 74 36.48648649 75 36.00000000 76 36.84210526 77 36.36363636 78 35.89743590 79 35.44303797 80 36.25000000 81 35.80246914 82 36.58536585 83 36.14457831 84 35.71428571 85 35.29411765 86 34.88372093 87 34.48275862 88 34.09090909 89 34.83146067 90 34.44444444 91 34.06593407 92 33.69565217 93 33.33333333 94 32.97872340 95 32.63157895 96 33.33333333 97 32.98969072 98 32.65306122 99 33.33333333 100 33.00000000 101 33.66336634 102 33.33333333 103 33.98058252 104 33.65384615 105 33.33333333 106 33.01886792 107 33.64485981 108 34.25925926 109 33.94495413 110 34.54545455 111 34.23423423 112 34.82142857 113 35.39823009 114 35.08771930 115 34.78260870 116 34.48275862 117 34.18803419 118 33.89830508 119 33.61344538 120 34.16666667 121 34.71074380 122 34.42622951 123 34.14634146 124 34.67741935 125 35.20000000 126 35.71428571 127 35.43307087 128 35.15625000 129 35.65891473 130 35.38461538 131 35.11450382 132 34.84848485 133 34.58646617 134 34.32835821 135 34.81481481 136 34.55882353 137 35.03649635 138 35.50724638 139 35.97122302 140 35.71428571 141 35.46099291 142 35.21126761 143 34.96503497 144 35.41666667 145 35.17241379 146 34.93150685 147 34.69387755 148 34.45945946 149 34.22818792 150 34.00000000 151 34.43708609 152 34.21052632 153 33.98692810 154 33.76623377 155 33.54838710 156 33.97435897 157 33.75796178 158 34.17721519 159 33.96226415 160 33.75000000 161 33.54037267 162 33.95061728 163 33.74233129 164 33.53658537 165 33.93939394 166 34.33734940 167 34.13173653 168 34.52380952 169 34.31952663 170 34.11764706 171 33.91812865 172 33.72093023 173 33.52601156 174 33.90804598 175 33.71428571 176 34.09090909 177 33.89830508 178 33.70786517 179 34.07821229 180 33.88888889 181 33.70165746 182 33.51648352 183 33.33333333 184 33.15217391 185 32.97297297 186 33.33333333 187 33.15508021 188 32.97872340 189 33.33333333 190 33.15789474 191 32.98429319 192 32.81250000 193 32.64248705 194 32.98969072 195 32.82051282 196 32.65306122 197 32.99492386 198 32.82828283 199 32.66331658 200 33.00000000 201 32.83582090 202 32.67326733 203 32.51231527 204 32.35294118 205 32.19512195 206 32.03883495 207 32.36714976 208 32.21153846 209 32.05741627 210 31.90476190 211 31.75355450 212 31.60377358 213 31.45539906 214 31.30841121 215 31.16279070 216 31.48148148 217 31.33640553 218 31.65137615 219 31.50684932 220 31.36363636 221 31.22171946 222 31.08108108 223 30.94170404 224 31.25000000 225 31.11111111 226 30.97345133 227 31.27753304 228 31.14035088 229 31.00436681 230 30.86956522 231 30.73593074 232 30.60344828 233 30.90128755 234 31.19658120 235 31.48936170 236 31.77966102 237 31.64556962 238 31.93277311 239 31.79916318 240 31.66666667 241 31.53526971 242 31.40495868 243 31.27572016 244 31.14754098 245 31.02040816 246 30.89430894 247 30.76923077 248 30.64516129 249 30.52208835 250 30.40000000 251 30.67729084 252 30.95238095 253 30.83003953 254 30.70866142 255 30.98039216 256 30.85937500 257 30.73929961 258 30.62015504 259 30.50193050 260 30.38461538 261 30.65134100 262 30.53435115 263 30.41825095 264 30.68181818 265 30.94339623 266 30.82706767 267 30.71161049 268 30.59701493 269 30.48327138 270 30.37037037 271 30.25830258 272 30.14705882 273 30.40293040 274 30.29197080 275 30.18181818 276 30.07246377 277 29.96389892 278 30.21582734 279 30.10752688 280 30.35714286 281 30.60498221 282 30.49645390 283 30.74204947 284 30.63380282 285 30.52631579 286 30.76923077 287 31.01045296 288 31.25000000 289 31.14186851 290 31.03448276 291 31.27147766 292 31.50684932 293 31.74061433 294 31.63265306 295 31.86440678 296 32.09459459 297 32.32323232 298 32.21476510 299 32.10702341 300 32.00000000 301 31.89368771 302 31.78807947 303 31.68316832 304 31.57894737 305 31.47540984 306 31.37254902 307 31.27035831 308 31.49350649 309 31.39158576 310 31.29032258 311 31.18971061 312 31.41025641 313 31.30990415 314 31.21019108 315 31.11111111 316 31.01265823 317 31.23028391 318 31.13207547 319 31.03448276 320 30.93750000 321 30.84112150 322 31.05590062 323 30.95975232 324 30.86419753 325 30.76923077 326 30.67484663 327 30.88685015 328 31.09756098 329 31.00303951 330 31.21212121 331 31.11782477 332 31.32530120 333 31.53153153 334 31.43712575 335 31.64179104 336 31.84523810 337 31.75074184 338 31.65680473 339 31.85840708 340 31.76470588 341 31.96480938 342 32.16374269 343 32.06997085 344 32.26744186 345 32.46376812 346 32.36994220 347 32.56484150 348 32.47126437 349 32.37822350 350 32.28571429 351 32.19373219 352 32.38636364 353 32.29461756 354 32.20338983 355 32.39436620 356 32.30337079 357 32.49299720 358 32.40223464 359 32.31197772 360 32.22222222 361 32.13296399 362 32.04419890 363 31.95592287 364 31.86813187 365 31.78082192 366 31.69398907 367 31.88010899 368 31.79347826 369 31.70731707 370 31.62162162 371 31.53638814 372 31.72043011 373 31.63538874 374 31.81818182 375 31.73333333 376 31.64893617 377 31.83023873 378 31.74603175 379 31.66226913 380 31.84210526 381 31.75853018 382 31.93717277 383 31.85378590 384 31.77083333 385 31.68831169 386 31.60621762 387 31.52454780 388 31.70103093 389 31.61953728 390 31.53846154 391 31.71355499 392 31.63265306 393 31.80661578 394 31.97969543 395 31.89873418 396 32.07070707 397 32.24181360 398 32.16080402 399 32.08020050 400 32.00000000 401 32.16957606 402 32.33830846 403 32.25806452 404 32.17821782 405 32.09876543 406 32.01970443 407 31.94103194 408 31.86274510 409 31.78484108 410 31.70731707 411 31.63017032 412 31.79611650 413 31.96125908 414 32.12560386 415 32.28915663 416 32.21153846 417 32.13429257 418 32.05741627 419 31.98090692 420 32.14285714 421 32.30403800 422 32.22748815 423 32.15130024 424 32.07547170 425 32.00000000 426 31.92488263 427 31.85011710 428 31.77570093 429 31.70163170 430 31.62790698 431 31.55452436 432 31.71296296 433 31.87066975 434 32.02764977 435 31.95402299 436 32.11009174 437 32.03661327 438 32.19178082 439 32.11845103 440 32.27272727 441 32.19954649 442 32.12669683 443 32.05417607 444 32.20720721 445 32.13483146 446 32.06278027 447 31.99105145 448 31.91964286 449 32.07126949 450 32.22222222 451 32.15077605 452 32.07964602 453 32.22958057 454 32.15859031 455 32.08791209 456 32.01754386 457 31.94748359 458 31.87772926 459 31.80827887 460 31.73913043 461 31.67028200 462 31.81818182 463 31.74946004 464 31.89655172 465 32.04301075 466 31.97424893 467 31.90578158 468 32.05128205 469 31.98294243 470 31.91489362 471 31.84713376 472 31.99152542 473 31.92389006 474 31.85654008 475 32.00000000 476 31.93277311 477 31.86582809 478 31.79916318 479 31.73277662 480 31.66666667 481 31.60083160 482 31.74273859 483 31.67701863 484 31.81818182 485 31.95876289 486 31.89300412 487 32.03285421 488 31.96721311 489 32.10633947 490 32.04081633 491 32.17922607 492 32.31707317 493 32.25152130 494 32.38866397 495 32.52525253 496 32.45967742 497 32.39436620 498 32.32931727 499 32.46492986 500 32.40000000 501 32.53493014 502 32.47011952 503 32.40556660 504 32.53968254 505 32.47524752 506 32.41106719 507 32.34714004 508 32.48031496 509 32.61296660 510 32.74509804 511 32.68101761 512 32.61718750 513 32.55360624 514 32.49027237 515 32.62135922 516 32.75193798 517 32.88201161 518 32.81853282 519 32.75529865 520 32.69230769 521 32.62955854 522 32.56704981 523 32.50478011 524 32.44274809 525 32.38095238 526 32.31939163 527 32.44781784 528 32.38636364 529 32.32514178 530 32.26415094 531 32.20338983 532 32.33082707 533 32.45778612 534 32.39700375 535 32.33644860 536 32.27611940 537 32.21601490 538 32.15613383 539 32.09647495 540 32.03703704 541 31.97781885 542 31.91881919 543 31.86003683 544 31.80147059 545 31.74311927 546 31.86813187 547 31.99268739 548 32.11678832 549 32.05828780 550 32.00000000 551 31.94192377 552 32.06521739 553 32.18806510 554 32.12996390 555 32.07207207 556 32.01438849 557 31.95691203 558 31.89964158 559 31.84257603 560 31.96428571 561 32.08556150 562 32.02846975 563 31.97158082 564 32.09219858 565 32.21238938 566 32.33215548 567 32.45149912 568 32.57042254 569 32.51318102 570 32.45614035 571 32.57443082 572 32.51748252 573 32.46073298 574 32.57839721 575 32.69565217 576 32.81250000 577 32.92894281 578 33.04498270 579 33.16062176 580 33.27586207 581 33.39070568 582 33.50515464 583 33.44768439 584 33.56164384 585 33.50427350 586 33.61774744 587 33.56047700 588 33.50340136 589 33.44651952 590 33.55932203 591 33.50253807 592 33.44594595 593 33.55817875 594 33.67003367 595 33.61344538 596 33.55704698 597 33.50083752 598 33.44481605 599 33.55592654 600 33.50000000 601 33.61064892 602 33.72093023 603 33.83084577 604 33.77483444 605 33.71900826 606 33.82838284 607 33.77265239 608 33.71710526 609 33.82594417 610 33.77049180 611 33.71522095 612 33.82352941 613 33.76835237 614 33.71335505 615 33.65853659 616 33.60389610 617 33.71150729 618 33.65695793 619 33.76413570 620 33.70967742 621 33.65539452 622 33.76205788 623 33.70786517 624 33.65384615 625 33.60000000 626 33.54632588 627 33.65231260 628 33.75796178 629 33.86327504 630 33.80952381 631 33.91442155 632 33.86075949 633 33.80726698 634 33.91167192 635 33.85826772 636 33.80503145 637 33.75196232 638 33.69905956 639 33.64632238 640 33.75000000 641 33.69734789 642 33.80062305 643 33.74805599 644 33.69565217 645 33.79844961 646 33.74613003 647 33.84853168 648 33.95061728 649 33.89830508 650 34.00000000 651 34.10138249 652 34.20245399 653 34.30321593 654 34.25076453 655 34.19847328 656 34.29878049 657 34.24657534 658 34.19452888 659 34.29438543 660 34.24242424 661 34.19062027 662 34.13897281 663 34.08748115 664 34.03614458 665 34.13533835 666 34.23423423 667 34.18290855 668 34.13173653 669 34.08071749 670 34.17910448 671 34.12816692 672 34.07738095 673 34.02674591 674 33.97626113 675 34.07407407 676 34.17159763 677 34.12112260 678 34.07079646 679 34.02061856 680 33.97058824 681 33.92070485 682 33.87096774 683 33.82137628 684 33.91812865 685 33.86861314 686 33.81924198 687 33.77001456 688 33.72093023 689 33.81712627 690 33.76811594 691 33.86396527 692 33.95953757 693 34.05483405 694 34.00576369 695 33.95683453 696 34.05172414 697 34.00286944 698 33.95415473 699 34.04864092 700 34.14285714 701 34.09415121 702 34.04558405 703 33.99715505 704 33.94886364 705 33.90070922 706 33.99433428 707 33.94625177 708 33.89830508 709 33.85049365 710 33.80281690 711 33.75527426 712 33.70786517 713 33.66058906 714 33.75350140 715 33.84615385 716 33.79888268 717 33.89121339 718 33.98328691 719 34.07510431 720 34.02777778 721 33.98058252 722 33.93351801 723 33.88658368 724 33.97790055 725 33.93103448 726 34.02203857 727 33.97524072 728 33.92857143 729 33.88203018 730 33.97260274 731 33.92612859 732 34.01639344 733 33.96998636 734 33.92370572 735 33.87755102 736 33.83152174 737 33.92130258 738 33.87533875 739 33.96481732 740 33.91891892 741 34.00809717 742 33.96226415 743 33.91655451 744 34.00537634 745 33.95973154 746 34.04825737 747 34.13654618 748 34.09090909 749 34.17890521 750 34.26666667 Final result: 34.2667 +/- 1.7342 Random chance: 19.8992 +/- 1.4588