common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 869 tasks in prompt multiple_choice_score: selecting 750 random tasks from 869 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 100.00000000 3 100.00000000 4 75.00000000 5 80.00000000 6 83.33333333 7 85.71428571 8 87.50000000 9 77.77777778 10 80.00000000 11 72.72727273 12 75.00000000 13 76.92307692 14 78.57142857 15 80.00000000 16 81.25000000 17 82.35294118 18 77.77777778 19 78.94736842 20 75.00000000 21 76.19047619 22 77.27272727 23 73.91304348 24 75.00000000 25 72.00000000 26 69.23076923 27 70.37037037 28 71.42857143 29 72.41379310 30 73.33333333 31 70.96774194 32 71.87500000 33 69.69696970 34 67.64705882 35 65.71428571 36 63.88888889 37 64.86486486 38 65.78947368 39 66.66666667 40 67.50000000 41 68.29268293 42 66.66666667 43 65.11627907 44 63.63636364 45 64.44444444 46 65.21739130 47 65.95744681 48 64.58333333 49 65.30612245 50 64.00000000 51 64.70588235 52 65.38461538 53 66.03773585 54 64.81481481 55 63.63636364 56 64.28571429 57 64.91228070 58 65.51724138 59 64.40677966 60 65.00000000 61 65.57377049 62 64.51612903 63 65.07936508 64 65.62500000 65 64.61538462 66 65.15151515 67 65.67164179 68 66.17647059 69 66.66666667 70 65.71428571 71 64.78873239 72 63.88888889 73 64.38356164 74 64.86486486 75 65.33333333 76 64.47368421 77 63.63636364 78 64.10256410 79 63.29113924 80 63.75000000 81 62.96296296 82 63.41463415 83 62.65060241 84 63.09523810 85 62.35294118 86 62.79069767 87 62.06896552 88 62.50000000 89 62.92134831 90 63.33333333 91 63.73626374 92 64.13043478 93 63.44086022 94 62.76595745 95 63.15789474 96 63.54166667 97 63.91752577 98 64.28571429 99 64.64646465 100 65.00000000 101 64.35643564 102 63.72549020 103 63.10679612 104 62.50000000 105 62.85714286 106 62.26415094 107 62.61682243 108 62.03703704 109 62.38532110 110 62.72727273 111 63.06306306 112 63.39285714 113 63.71681416 114 64.03508772 115 64.34782609 116 63.79310345 117 64.10256410 118 64.40677966 119 63.86554622 120 64.16666667 121 63.63636364 122 63.93442623 123 64.22764228 124 63.70967742 125 64.00000000 126 64.28571429 127 63.77952756 128 64.06250000 129 63.56589147 130 63.07692308 131 63.35877863 132 63.63636364 133 63.90977444 134 63.43283582 135 62.96296296 136 63.23529412 137 63.50364964 138 63.76811594 139 64.02877698 140 64.28571429 141 64.53900709 142 64.78873239 143 65.03496503 144 65.27777778 145 65.51724138 146 65.06849315 147 64.62585034 148 64.86486486 149 65.10067114 150 65.33333333 151 64.90066225 152 65.13157895 153 65.35947712 154 64.93506494 155 65.16129032 156 65.38461538 157 65.60509554 158 65.18987342 159 65.40880503 160 65.62500000 161 65.83850932 162 66.04938272 163 66.25766871 164 65.85365854 165 65.45454545 166 65.66265060 167 65.26946108 168 65.47619048 169 65.08875740 170 65.29411765 171 64.91228070 172 65.11627907 173 65.31791908 174 65.51724138 175 65.14285714 176 65.34090909 177 65.53672316 178 65.16853933 179 65.36312849 180 65.55555556 181 65.19337017 182 65.38461538 183 65.02732240 184 65.21739130 185 65.40540541 186 65.59139785 187 65.77540107 188 65.95744681 189 65.60846561 190 65.26315789 191 65.44502618 192 65.62500000 193 65.28497409 194 65.46391753 195 65.64102564 196 65.81632653 197 65.98984772 198 66.16161616 199 65.82914573 200 66.00000000 201 66.16915423 202 66.33663366 203 66.50246305 204 66.66666667 205 66.34146341 206 66.01941748 207 65.70048309 208 65.86538462 209 66.02870813 210 66.19047619 211 66.35071090 212 66.03773585 213 66.19718310 214 66.35514019 215 66.51162791 216 66.66666667 217 66.35944700 218 66.51376147 219 66.21004566 220 65.90909091 221 65.61085973 222 65.76576577 223 65.47085202 224 65.62500000 225 65.33333333 226 65.04424779 227 65.19823789 228 65.35087719 229 65.50218341 230 65.65217391 231 65.36796537 232 65.08620690 233 65.23605150 234 65.38461538 235 65.10638298 236 65.25423729 237 65.40084388 238 65.54621849 239 65.69037657 240 65.83333333 241 65.97510373 242 66.11570248 243 66.25514403 244 65.98360656 245 66.12244898 246 66.26016260 247 66.39676113 248 66.53225806 249 66.66666667 250 66.80000000 251 66.93227092 252 67.06349206 253 67.19367589 254 66.92913386 255 67.05882353 256 67.18750000 257 66.92607004 258 67.05426357 259 66.79536680 260 66.92307692 261 67.04980843 262 67.17557252 263 67.30038023 264 67.42424242 265 67.16981132 266 66.91729323 267 67.04119850 268 67.16417910 269 67.28624535 270 67.40740741 271 67.52767528 272 67.27941176 273 67.03296703 274 66.78832117 275 66.54545455 276 66.66666667 277 66.78700361 278 66.54676259 279 66.66666667 280 66.78571429 281 66.90391459 282 67.02127660 283 66.78445230 284 66.90140845 285 67.01754386 286 67.13286713 287 67.24738676 288 67.36111111 289 67.12802768 290 67.24137931 291 67.35395189 292 67.46575342 293 67.57679181 294 67.34693878 295 67.45762712 296 67.56756757 297 67.67676768 298 67.78523490 299 67.89297659 300 68.00000000 301 67.77408638 302 67.88079470 303 67.98679868 304 68.09210526 305 67.86885246 306 67.64705882 307 67.75244300 308 67.85714286 309 67.96116505 310 68.06451613 311 68.16720257 312 68.26923077 313 68.37060703 314 68.15286624 315 67.93650794 316 68.03797468 317 68.13880126 318 67.92452830 319 67.71159875 320 67.50000000 321 67.60124611 322 67.70186335 323 67.80185759 324 67.59259259 325 67.69230769 326 67.79141104 327 67.58409786 328 67.68292683 329 67.78115502 330 67.57575758 331 67.67371601 332 67.77108434 333 67.86786787 334 67.66467066 335 67.46268657 336 67.26190476 337 67.06231454 338 66.86390533 339 66.96165192 340 67.05882353 341 66.86217009 342 66.66666667 343 66.76384840 344 66.86046512 345 66.66666667 346 66.47398844 347 66.28242075 348 66.09195402 349 66.18911175 350 66.28571429 351 66.38176638 352 66.47727273 353 66.57223796 354 66.38418079 355 66.19718310 356 66.29213483 357 66.38655462 358 66.48044693 359 66.29526462 360 66.38888889 361 66.48199446 362 66.57458564 363 66.39118457 364 66.20879121 365 66.30136986 366 66.12021858 367 66.21253406 368 66.30434783 369 66.39566396 370 66.48648649 371 66.57681941 372 66.66666667 373 66.75603217 374 66.84491979 375 66.66666667 376 66.75531915 377 66.57824934 378 66.66666667 379 66.49076517 380 66.57894737 381 66.66666667 382 66.49214660 383 66.31853786 384 66.40625000 385 66.23376623 386 66.32124352 387 66.14987080 388 65.97938144 389 65.80976864 390 65.64102564 391 65.47314578 392 65.56122449 393 65.64885496 394 65.73604061 395 65.82278481 396 65.65656566 397 65.49118388 398 65.32663317 399 65.41353383 400 65.50000000 401 65.33665835 402 65.17412935 403 65.26054591 404 65.34653465 405 65.43209877 406 65.27093596 407 65.35626536 408 65.44117647 409 65.28117359 410 65.36585366 411 65.45012165 412 65.53398058 413 65.37530266 414 65.45893720 415 65.30120482 416 65.14423077 417 64.98800959 418 65.07177033 419 64.91646778 420 65.00000000 421 64.84560570 422 64.69194313 423 64.77541371 424 64.62264151 425 64.70588235 426 64.78873239 427 64.87119438 428 64.71962617 429 64.80186480 430 64.88372093 431 64.96519722 432 65.04629630 433 64.89607390 434 64.97695853 435 65.05747126 436 65.13761468 437 65.21739130 438 65.06849315 439 65.14806378 440 65.00000000 441 65.07936508 442 64.93212670 443 64.78555305 444 64.63963964 445 64.71910112 446 64.79820628 447 64.87695749 448 64.95535714 449 65.03340757 450 64.88888889 451 64.96674058 452 65.04424779 453 65.12141280 454 64.97797357 455 64.83516484 456 64.91228070 457 64.98905908 458 65.06550218 459 65.14161220 460 65.21739130 461 65.29284165 462 65.36796537 463 65.22678186 464 65.08620690 465 65.16129032 466 65.23605150 467 65.31049251 468 65.38461538 469 65.24520256 470 65.31914894 471 65.39278132 472 65.46610169 473 65.32769556 474 65.40084388 475 65.47368421 476 65.33613445 477 65.40880503 478 65.48117155 479 65.55323591 480 65.62500000 481 65.48856549 482 65.56016598 483 65.63146998 484 65.70247934 485 65.77319588 486 65.84362140 487 65.70841889 488 65.77868852 489 65.84867076 490 65.71428571 491 65.58044807 492 65.65040650 493 65.51724138 494 65.58704453 495 65.45454545 496 65.52419355 497 65.59356137 498 65.66265060 499 65.73146293 500 65.80000000 501 65.66866267 502 65.73705179 503 65.80516899 504 65.87301587 505 65.94059406 506 65.81027668 507 65.68047337 508 65.74803150 509 65.61886051 510 65.68627451 511 65.55772994 512 65.42968750 513 65.30214425 514 65.36964981 515 65.24271845 516 65.11627907 517 65.18375242 518 65.25096525 519 65.31791908 520 65.19230769 521 65.25911708 522 65.32567050 523 65.20076482 524 65.26717557 525 65.14285714 526 65.01901141 527 64.89563567 528 64.77272727 529 64.65028355 530 64.71698113 531 64.78342750 532 64.84962406 533 64.91557223 534 64.79400749 535 64.67289720 536 64.73880597 537 64.80446927 538 64.68401487 539 64.74953618 540 64.62962963 541 64.69500924 542 64.76014760 543 64.82504604 544 64.88970588 545 64.95412844 546 64.83516484 547 64.71663620 548 64.78102190 549 64.66302368 550 64.72727273 551 64.60980036 552 64.49275362 553 64.55696203 554 64.62093863 555 64.68468468 556 64.56834532 557 64.63195691 558 64.51612903 559 64.57960644 560 64.64285714 561 64.70588235 562 64.76868327 563 64.83126110 564 64.71631206 565 64.60176991 566 64.66431095 567 64.55026455 568 64.61267606 569 64.49912127 570 64.38596491 571 64.44833625 572 64.33566434 573 64.39790576 574 64.45993031 575 64.34782609 576 64.40972222 577 64.47140381 578 64.53287197 579 64.59412781 580 64.48275862 581 64.54388985 582 64.43298969 583 64.32246998 584 64.38356164 585 64.44444444 586 64.33447099 587 64.39522998 588 64.45578231 589 64.51612903 590 64.57627119 591 64.63620981 592 64.69594595 593 64.75548061 594 64.81481481 595 64.87394958 596 64.76510067 597 64.65661642 598 64.54849498 599 64.60767947 600 64.50000000 601 64.55906822 602 64.45182724 603 64.34494196 604 64.40397351 605 64.46280992 606 64.52145215 607 64.57990115 608 64.63815789 609 64.53201970 610 64.59016393 611 64.48445172 612 64.54248366 613 64.43719413 614 64.49511401 615 64.55284553 616 64.44805195 617 64.50567261 618 64.40129450 619 64.29725363 620 64.35483871 621 64.25120773 622 64.14790997 623 64.20545746 624 64.10256410 625 64.00000000 626 63.89776358 627 63.95534290 628 64.01273885 629 63.91096979 630 63.96825397 631 64.02535658 632 63.92405063 633 63.82306477 634 63.88012618 635 63.93700787 636 63.99371069 637 63.89324961 638 63.94984326 639 64.00625978 640 63.90625000 641 63.96255850 642 63.86292835 643 63.91912908 644 63.97515528 645 63.87596899 646 63.93188854 647 63.98763524 648 63.88888889 649 63.79044684 650 63.84615385 651 63.90168971 652 63.95705521 653 63.85911179 654 63.76146789 655 63.66412214 656 63.56707317 657 63.47031963 658 63.52583587 659 63.42943854 660 63.48484848 661 63.38880484 662 63.44410876 663 63.49924585 664 63.55421687 665 63.60902256 666 63.51351351 667 63.56821589 668 63.47305389 669 63.37817638 670 63.43283582 671 63.33830104 672 63.39285714 673 63.29866270 674 63.20474777 675 63.11111111 676 63.01775148 677 63.07237814 678 63.12684366 679 63.18114875 680 63.23529412 681 63.14243759 682 63.04985337 683 62.95754026 684 62.86549708 685 62.91970803 686 62.82798834 687 62.73653566 688 62.79069767 689 62.69956459 690 62.75362319 691 62.80752533 692 62.86127168 693 62.91486291 694 62.82420749 695 62.73381295 696 62.64367816 697 62.69727403 698 62.60744986 699 62.51788269 700 62.57142857 701 62.62482168 702 62.67806268 703 62.73115220 704 62.78409091 705 62.69503546 706 62.74787535 707 62.80056577 708 62.85310734 709 62.90550071 710 62.95774648 711 63.00984529 712 63.06179775 713 63.11360449 714 63.02521008 715 62.93706294 716 62.98882682 717 62.90097629 718 62.81337047 719 62.72600834 720 62.77777778 721 62.82940361 722 62.88088643 723 62.93222683 724 62.84530387 725 62.89655172 726 62.80991736 727 62.72352132 728 62.77472527 729 62.82578875 730 62.87671233 731 62.92749658 732 62.97814208 733 63.02864939 734 63.07901907 735 62.99319728 736 63.04347826 737 62.95793758 738 63.00813008 739 62.92286874 740 62.97297297 741 63.02294197 742 63.07277628 743 63.12247645 744 63.03763441 745 63.08724832 746 63.13672922 747 63.05220884 748 62.96791444 749 62.88384513 750 62.93333333 Final result: 62.9333 +/- 1.7648 Random chance: 25.0083 +/- 1.5824