common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 869 tasks in prompt multiple_choice_score: selecting 750 random tasks from 869 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 100.00000000 3 100.00000000 4 75.00000000 5 80.00000000 6 83.33333333 7 71.42857143 8 75.00000000 9 66.66666667 10 70.00000000 11 63.63636364 12 66.66666667 13 69.23076923 14 71.42857143 15 73.33333333 16 75.00000000 17 76.47058824 18 72.22222222 19 73.68421053 20 70.00000000 21 71.42857143 22 72.72727273 23 69.56521739 24 70.83333333 25 68.00000000 26 65.38461538 27 66.66666667 28 64.28571429 29 62.06896552 30 63.33333333 31 61.29032258 32 59.37500000 33 57.57575758 34 55.88235294 35 54.28571429 36 52.77777778 37 54.05405405 38 52.63157895 39 53.84615385 40 55.00000000 41 56.09756098 42 54.76190476 43 53.48837209 44 52.27272727 45 53.33333333 46 54.34782609 47 55.31914894 48 54.16666667 49 55.10204082 50 54.00000000 51 54.90196078 52 55.76923077 53 56.60377358 54 55.55555556 55 54.54545455 56 55.35714286 57 56.14035088 58 56.89655172 59 57.62711864 60 58.33333333 61 57.37704918 62 56.45161290 63 57.14285714 64 57.81250000 65 56.92307692 66 57.57575758 67 58.20895522 68 58.82352941 69 59.42028986 70 58.57142857 71 57.74647887 72 56.94444444 73 57.53424658 74 56.75675676 75 57.33333333 76 56.57894737 77 55.84415584 78 56.41025641 79 55.69620253 80 56.25000000 81 56.79012346 82 57.31707317 83 56.62650602 84 57.14285714 85 57.64705882 86 58.13953488 87 57.47126437 88 57.95454545 89 58.42696629 90 58.88888889 91 59.34065934 92 59.78260870 93 59.13978495 94 58.51063830 95 58.94736842 96 59.37500000 97 58.76288660 98 59.18367347 99 59.59595960 100 60.00000000 101 60.39603960 102 59.80392157 103 59.22330097 104 59.61538462 105 60.00000000 106 59.43396226 107 59.81308411 108 59.25925926 109 58.71559633 110 59.09090909 111 59.45945946 112 59.82142857 113 60.17699115 114 59.64912281 115 60.00000000 116 59.48275862 117 59.82905983 118 60.16949153 119 59.66386555 120 60.00000000 121 59.50413223 122 59.83606557 123 60.16260163 124 59.67741935 125 60.00000000 126 60.31746032 127 60.62992126 128 60.93750000 129 60.46511628 130 60.76923077 131 61.06870229 132 61.36363636 133 61.65413534 134 61.19402985 135 60.74074074 136 61.02941176 137 61.31386861 138 61.59420290 139 61.87050360 140 62.14285714 141 62.41134752 142 61.97183099 143 62.23776224 144 62.50000000 145 62.75862069 146 62.32876712 147 61.90476190 148 62.16216216 149 62.41610738 150 62.66666667 151 62.25165563 152 62.50000000 153 62.74509804 154 62.33766234 155 61.93548387 156 62.17948718 157 62.42038217 158 62.02531646 159 62.26415094 160 62.50000000 161 62.73291925 162 62.96296296 163 63.19018405 164 62.80487805 165 62.42424242 166 62.65060241 167 62.27544910 168 62.50000000 169 62.13017751 170 61.76470588 171 61.98830409 172 62.20930233 173 62.42774566 174 62.64367816 175 62.28571429 176 62.50000000 177 62.71186441 178 62.92134831 179 63.12849162 180 63.33333333 181 62.98342541 182 63.18681319 183 62.84153005 184 63.04347826 185 63.24324324 186 63.44086022 187 63.63636364 188 63.82978723 189 63.49206349 190 63.15789474 191 63.35078534 192 63.54166667 193 63.21243523 194 63.40206186 195 63.58974359 196 63.77551020 197 63.95939086 198 64.14141414 199 63.81909548 200 64.00000000 201 64.17910448 202 63.86138614 203 64.03940887 204 64.21568627 205 63.90243902 206 63.59223301 207 63.76811594 208 63.94230769 209 64.11483254 210 64.28571429 211 64.45497630 212 64.15094340 213 64.31924883 214 64.48598131 215 64.65116279 216 64.35185185 217 64.51612903 218 64.67889908 219 64.84018265 220 65.00000000 221 64.70588235 222 64.86486486 223 64.57399103 224 64.28571429 225 64.00000000 226 63.71681416 227 63.87665198 228 64.03508772 229 64.19213974 230 64.34782609 231 64.06926407 232 63.79310345 233 63.94849785 234 63.67521368 235 63.40425532 236 63.13559322 237 62.86919831 238 63.02521008 239 63.17991632 240 63.33333333 241 63.48547718 242 63.63636364 243 63.78600823 244 63.52459016 245 63.67346939 246 63.82113821 247 63.96761134 248 64.11290323 249 64.25702811 250 64.40000000 251 64.54183267 252 64.68253968 253 64.82213439 254 64.56692913 255 64.70588235 256 64.45312500 257 64.20233463 258 64.34108527 259 64.09266409 260 64.23076923 261 64.36781609 262 64.50381679 263 64.63878327 264 64.77272727 265 64.52830189 266 64.28571429 267 64.41947566 268 64.55223881 269 64.31226766 270 64.44444444 271 64.57564576 272 64.33823529 273 64.10256410 274 63.86861314 275 64.00000000 276 64.13043478 277 64.25992780 278 64.02877698 279 64.15770609 280 64.28571429 281 64.41281139 282 64.53900709 283 64.31095406 284 64.43661972 285 64.56140351 286 64.68531469 287 64.80836237 288 64.93055556 289 64.70588235 290 64.82758621 291 64.94845361 292 65.06849315 293 65.18771331 294 64.96598639 295 65.08474576 296 65.20270270 297 65.31986532 298 65.43624161 299 65.55183946 300 65.66666667 301 65.44850498 302 65.56291391 303 65.67656766 304 65.78947368 305 65.90163934 306 65.68627451 307 65.79804560 308 65.90909091 309 66.01941748 310 66.12903226 311 66.23794212 312 66.02564103 313 66.13418530 314 65.92356688 315 65.71428571 316 65.82278481 317 65.93059937 318 65.72327044 319 65.51724138 320 65.31250000 321 65.42056075 322 65.21739130 323 65.32507740 324 65.12345679 325 65.23076923 326 65.33742331 327 65.13761468 328 65.24390244 329 65.34954407 330 65.15151515 331 65.25679758 332 65.36144578 333 65.46546547 334 65.56886228 335 65.37313433 336 65.17857143 337 65.28189911 338 65.08875740 339 65.19174041 340 65.29411765 341 65.39589443 342 65.20467836 343 65.30612245 344 65.40697674 345 65.21739130 346 65.02890173 347 64.84149856 348 64.65517241 349 64.75644699 350 64.85714286 351 64.95726496 352 65.05681818 353 65.15580737 354 64.97175141 355 64.78873239 356 64.88764045 357 64.98599440 358 65.08379888 359 64.90250696 360 65.00000000 361 65.09695291 362 65.19337017 363 65.28925620 364 65.38461538 365 65.47945205 366 65.30054645 367 65.12261580 368 65.21739130 369 65.31165312 370 65.40540541 371 65.22911051 372 65.32258065 373 65.41554960 374 65.50802139 375 65.33333333 376 65.15957447 377 64.98673740 378 65.07936508 379 64.90765172 380 65.00000000 381 65.09186352 382 64.92146597 383 64.75195822 384 64.84375000 385 64.67532468 386 64.76683938 387 64.59948320 388 64.43298969 389 64.26735219 390 64.10256410 391 63.93861893 392 64.03061224 393 64.12213740 394 64.21319797 395 64.30379747 396 64.14141414 397 64.23173804 398 64.07035176 399 64.16040100 400 64.25000000 401 64.33915212 402 64.42786070 403 64.51612903 404 64.60396040 405 64.69135802 406 64.77832512 407 64.86486486 408 64.95098039 409 64.79217604 410 64.87804878 411 64.96350365 412 65.04854369 413 65.13317191 414 65.21739130 415 65.06024096 416 64.90384615 417 64.98800959 418 65.07177033 419 64.91646778 420 65.00000000 421 64.84560570 422 64.69194313 423 64.77541371 424 64.62264151 425 64.70588235 426 64.78873239 427 64.87119438 428 64.71962617 429 64.80186480 430 64.88372093 431 64.96519722 432 65.04629630 433 64.89607390 434 64.97695853 435 65.05747126 436 65.13761468 437 65.21739130 438 65.29680365 439 65.37585421 440 65.22727273 441 65.30612245 442 65.15837104 443 65.23702032 444 65.09009009 445 65.16853933 446 65.24663677 447 65.32438479 448 65.40178571 449 65.47884187 450 65.33333333 451 65.41019956 452 65.48672566 453 65.56291391 454 65.41850220 455 65.27472527 456 65.35087719 457 65.42669584 458 65.50218341 459 65.57734205 460 65.43478261 461 65.29284165 462 65.36796537 463 65.22678186 464 65.08620690 465 65.16129032 466 65.23605150 467 65.31049251 468 65.38461538 469 65.24520256 470 65.31914894 471 65.39278132 472 65.46610169 473 65.32769556 474 65.40084388 475 65.47368421 476 65.54621849 477 65.61844864 478 65.69037657 479 65.76200418 480 65.83333333 481 65.69646570 482 65.76763485 483 65.83850932 484 65.90909091 485 65.97938144 486 66.04938272 487 65.91375770 488 65.98360656 489 65.84867076 490 65.71428571 491 65.78411405 492 65.85365854 493 65.72008114 494 65.78947368 495 65.65656566 496 65.52419355 497 65.59356137 498 65.66265060 499 65.73146293 500 65.80000000 501 65.66866267 502 65.73705179 503 65.80516899 504 65.87301587 505 65.94059406 506 65.81027668 507 65.68047337 508 65.74803150 509 65.61886051 510 65.68627451 511 65.55772994 512 65.42968750 513 65.49707602 514 65.56420233 515 65.63106796 516 65.50387597 517 65.57059961 518 65.63706564 519 65.70327553 520 65.57692308 521 65.64299424 522 65.70881226 523 65.58317400 524 65.64885496 525 65.52380952 526 65.39923954 527 65.27514231 528 65.15151515 529 65.02835539 530 65.09433962 531 65.16007533 532 65.22556391 533 65.29080675 534 65.16853933 535 65.04672897 536 65.11194030 537 65.17690875 538 65.05576208 539 65.12059369 540 65.00000000 541 65.06469501 542 65.12915129 543 65.19337017 544 65.25735294 545 65.32110092 546 65.38461538 547 65.26508227 548 65.32846715 549 65.39162113 550 65.45454545 551 65.33575318 552 65.21739130 553 65.28028933 554 65.34296029 555 65.40540541 556 65.28776978 557 65.35008977 558 65.23297491 559 65.29516995 560 65.35714286 561 65.41889483 562 65.48042705 563 65.54174067 564 65.42553191 565 65.30973451 566 65.37102473 567 65.43209877 568 65.49295775 569 65.37785589 570 65.26315789 571 65.32399299 572 65.20979021 573 65.27050611 574 65.33101045 575 65.39130435 576 65.45138889 577 65.51126516 578 65.57093426 579 65.63039724 580 65.51724138 581 65.57659208 582 65.46391753 583 65.35162950 584 65.41095890 585 65.47008547 586 65.35836177 587 65.41737649 588 65.47619048 589 65.53480475 590 65.59322034 591 65.65143824 592 65.70945946 593 65.76728499 594 65.82491582 595 65.71428571 596 65.60402685 597 65.49413735 598 65.38461538 599 65.27545910 600 65.33333333 601 65.39101498 602 65.28239203 603 65.17412935 604 65.23178808 605 65.28925620 606 65.34653465 607 65.23887974 608 65.29605263 609 65.18883415 610 65.24590164 611 65.13911620 612 65.19607843 613 65.08972268 614 65.14657980 615 65.04065041 616 64.93506494 617 64.99189627 618 64.88673139 619 64.78190630 620 64.83870968 621 64.89533011 622 64.79099678 623 64.84751204 624 64.90384615 625 64.80000000 626 64.69648562 627 64.75279107 628 64.80891720 629 64.70588235 630 64.76190476 631 64.81774960 632 64.71518987 633 64.61295419 634 64.66876972 635 64.72440945 636 64.77987421 637 64.67817896 638 64.73354232 639 64.78873239 640 64.84375000 641 64.89859594 642 64.79750779 643 64.85225505 644 64.90683230 645 64.80620155 646 64.86068111 647 64.91499227 648 64.81481481 649 64.71494607 650 64.76923077 651 64.82334869 652 64.87730061 653 64.77794793 654 64.67889908 655 64.58015267 656 64.48170732 657 64.38356164 658 64.43768997 659 64.33990895 660 64.39393939 661 64.29652042 662 64.35045317 663 64.40422323 664 64.45783133 665 64.51127820 666 64.56456456 667 64.61769115 668 64.52095808 669 64.57399103 670 64.62686567 671 64.53055142 672 64.58333333 673 64.48736999 674 64.39169139 675 64.44444444 676 64.49704142 677 64.54948301 678 64.60176991 679 64.50662739 680 64.41176471 681 64.46402349 682 64.36950147 683 64.27525622 684 64.32748538 685 64.37956204 686 64.28571429 687 64.19213974 688 64.24418605 689 64.15094340 690 64.20289855 691 64.10998553 692 64.16184971 693 64.21356421 694 64.26512968 695 64.17266187 696 64.08045977 697 63.98852224 698 63.89684814 699 63.80543634 700 63.85714286 701 63.76604850 702 63.81766382 703 63.86913229 704 63.92045455 705 63.97163121 706 64.02266289 707 64.07355021 708 64.12429379 709 64.17489422 710 64.08450704 711 63.99437412 712 64.04494382 713 64.09537167 714 64.00560224 715 63.91608392 716 63.96648045 717 63.87726639 718 63.78830084 719 63.69958275 720 63.75000000 721 63.80027739 722 63.85041551 723 63.90041494 724 63.81215470 725 63.86206897 726 63.91184573 727 63.82393398 728 63.87362637 729 63.92318244 730 63.97260274 731 64.02188782 732 64.07103825 733 64.12005457 734 64.16893733 735 64.21768707 736 64.26630435 737 64.17910448 738 64.22764228 739 64.14073072 740 64.18918919 741 64.23751687 742 64.28571429 743 64.33378197 744 64.24731183 745 64.29530201 746 64.34316354 747 64.25702811 748 64.17112299 749 64.08544726 750 64.13333333 Final result: 64.1333 +/- 1.7525 Random chance: 25.0083 +/- 1.5824