common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 817 tasks in prompt multiple_choice_score: selecting 750 random tasks from 817 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 25.00000000 5 20.00000000 6 16.66666667 7 14.28571429 8 12.50000000 9 11.11111111 10 10.00000000 11 9.09090909 12 16.66666667 13 15.38461538 14 14.28571429 15 13.33333333 16 18.75000000 17 17.64705882 18 16.66666667 19 21.05263158 20 20.00000000 21 19.04761905 22 18.18181818 23 17.39130435 24 20.83333333 25 20.00000000 26 23.07692308 27 22.22222222 28 21.42857143 29 24.13793103 30 23.33333333 31 25.80645161 32 25.00000000 33 27.27272727 34 26.47058824 35 25.71428571 36 25.00000000 37 27.02702703 38 26.31578947 39 25.64102564 40 25.00000000 41 24.39024390 42 23.80952381 43 25.58139535 44 27.27272727 45 26.66666667 46 28.26086957 47 27.65957447 48 29.16666667 49 28.57142857 50 28.00000000 51 27.45098039 52 26.92307692 53 28.30188679 54 29.62962963 55 30.90909091 56 30.35714286 57 29.82456140 58 29.31034483 59 28.81355932 60 30.00000000 61 29.50819672 62 30.64516129 63 30.15873016 64 29.68750000 65 29.23076923 66 28.78787879 67 28.35820896 68 27.94117647 69 27.53623188 70 27.14285714 71 28.16901408 72 29.16666667 73 28.76712329 74 28.37837838 75 28.00000000 76 27.63157895 77 27.27272727 78 26.92307692 79 26.58227848 80 26.25000000 81 25.92592593 82 26.82926829 83 26.50602410 84 26.19047619 85 25.88235294 86 25.58139535 87 25.28735632 88 25.00000000 89 25.84269663 90 25.55555556 91 26.37362637 92 26.08695652 93 25.80645161 94 25.53191489 95 26.31578947 96 27.08333333 97 26.80412371 98 26.53061224 99 27.27272727 100 27.00000000 101 27.72277228 102 27.45098039 103 28.15533981 104 27.88461538 105 27.61904762 106 27.35849057 107 28.03738318 108 28.70370370 109 28.44036697 110 29.09090909 111 28.82882883 112 28.57142857 113 28.31858407 114 28.94736842 115 28.69565217 116 28.44827586 117 28.20512821 118 27.96610169 119 27.73109244 120 28.33333333 121 28.92561983 122 28.68852459 123 28.45528455 124 29.03225806 125 29.60000000 126 29.36507937 127 29.13385827 128 28.90625000 129 28.68217054 130 28.46153846 131 28.24427481 132 28.03030303 133 27.81954887 134 28.35820896 135 28.14814815 136 27.94117647 137 28.46715328 138 28.26086957 139 28.77697842 140 28.57142857 141 28.36879433 142 28.16901408 143 27.97202797 144 27.77777778 145 27.58620690 146 27.39726027 147 27.21088435 148 27.02702703 149 26.84563758 150 26.66666667 151 27.15231788 152 26.97368421 153 26.79738562 154 26.62337662 155 27.09677419 156 27.56410256 157 27.38853503 158 27.84810127 159 27.67295597 160 27.50000000 161 27.32919255 162 27.16049383 163 27.60736196 164 27.43902439 165 27.87878788 166 28.31325301 167 28.14371257 168 27.97619048 169 27.81065089 170 28.23529412 171 28.07017544 172 27.90697674 173 27.74566474 174 28.16091954 175 28.00000000 176 28.40909091 177 28.24858757 178 28.08988764 179 27.93296089 180 27.77777778 181 27.62430939 182 27.47252747 183 27.32240437 184 27.17391304 185 27.02702703 186 27.41935484 187 27.27272727 188 27.12765957 189 26.98412698 190 26.84210526 191 26.70157068 192 27.08333333 193 26.94300518 194 27.31958763 195 27.17948718 196 27.04081633 197 27.41116751 198 27.27272727 199 27.13567839 200 27.00000000 201 26.86567164 202 27.22772277 203 27.09359606 204 26.96078431 205 26.82926829 206 26.69902913 207 27.05314010 208 26.92307692 209 26.79425837 210 26.66666667 211 26.54028436 212 26.41509434 213 26.76056338 214 26.63551402 215 26.51162791 216 26.85185185 217 26.72811060 218 27.06422018 219 26.94063927 220 26.81818182 221 26.69683258 222 26.57657658 223 26.45739910 224 26.78571429 225 26.66666667 226 26.54867257 227 26.43171806 228 26.31578947 229 26.20087336 230 26.08695652 231 25.97402597 232 26.29310345 233 26.60944206 234 26.49572650 235 26.80851064 236 27.11864407 237 27.42616034 238 27.73109244 239 27.61506276 240 27.50000000 241 27.38589212 242 27.27272727 243 27.16049383 244 27.04918033 245 26.93877551 246 26.82926829 247 26.72064777 248 27.01612903 249 26.90763052 250 26.80000000 251 27.09163347 252 26.98412698 253 26.87747036 254 26.77165354 255 26.66666667 256 26.56250000 257 26.45914397 258 26.35658915 259 26.25482625 260 26.15384615 261 26.05363985 262 25.95419847 263 25.85551331 264 26.13636364 265 26.41509434 266 26.31578947 267 26.59176030 268 26.49253731 269 26.39405204 270 26.29629630 271 26.19926199 272 26.10294118 273 26.00732601 274 25.91240876 275 25.81818182 276 25.72463768 277 25.63176895 278 25.53956835 279 25.44802867 280 25.35714286 281 25.62277580 282 25.88652482 283 25.79505300 284 26.05633803 285 25.96491228 286 26.22377622 287 26.13240418 288 26.04166667 289 25.95155709 290 25.86206897 291 26.11683849 292 26.36986301 293 26.62116041 294 26.53061224 295 26.77966102 296 26.68918919 297 26.93602694 298 26.84563758 299 26.75585284 300 26.66666667 301 26.57807309 302 26.82119205 303 26.73267327 304 26.64473684 305 26.55737705 306 26.47058824 307 26.38436482 308 26.62337662 309 26.53721683 310 26.45161290 311 26.36655949 312 26.60256410 313 26.51757188 314 26.43312102 315 26.34920635 316 26.26582278 317 26.18296530 318 26.10062893 319 26.01880878 320 25.93750000 321 25.85669782 322 26.08695652 323 26.00619195 324 25.92592593 325 25.84615385 326 25.76687117 327 25.99388379 328 26.21951220 329 26.13981763 330 26.36363636 331 26.28398792 332 26.50602410 333 26.72672673 334 26.64670659 335 26.86567164 336 27.08333333 337 27.00296736 338 26.92307692 339 27.13864307 340 27.05882353 341 27.27272727 342 27.19298246 343 27.11370262 344 27.32558140 345 27.53623188 346 27.74566474 347 27.95389049 348 27.87356322 349 27.79369628 350 27.71428571 351 27.63532764 352 27.84090909 353 27.76203966 354 27.68361582 355 27.88732394 356 27.80898876 357 27.73109244 358 27.65363128 359 27.57660167 360 27.50000000 361 27.42382271 362 27.34806630 363 27.27272727 364 27.19780220 365 27.39726027 366 27.32240437 367 27.52043597 368 27.44565217 369 27.37127371 370 27.29729730 371 27.22371968 372 27.41935484 373 27.34584450 374 27.54010695 375 27.46666667 376 27.39361702 377 27.58620690 378 27.51322751 379 27.44063325 380 27.36842105 381 27.29658793 382 27.22513089 383 27.15404700 384 27.08333333 385 27.01298701 386 26.94300518 387 26.87338501 388 27.06185567 389 26.99228792 390 26.92307692 391 27.10997442 392 27.04081633 393 27.22646310 394 27.41116751 395 27.34177215 396 27.27272727 397 27.45591940 398 27.38693467 399 27.31829574 400 27.25000000 401 27.18204489 402 27.11442786 403 27.04714640 404 26.98019802 405 26.91358025 406 26.84729064 407 26.78132678 408 26.71568627 409 26.65036675 410 26.58536585 411 26.52068127 412 26.69902913 413 26.63438257 414 26.81159420 415 26.74698795 416 26.68269231 417 26.61870504 418 26.55502392 419 26.49164678 420 26.66666667 421 26.84085511 422 26.77725118 423 26.71394799 424 26.65094340 425 26.58823529 426 26.52582160 427 26.46370023 428 26.40186916 429 26.34032634 430 26.27906977 431 26.21809745 432 26.38888889 433 26.55889145 434 26.72811060 435 26.66666667 436 26.60550459 437 26.54462243 438 26.48401826 439 26.42369021 440 26.36363636 441 26.30385488 442 26.24434389 443 26.18510158 444 26.35135135 445 26.29213483 446 26.23318386 447 26.17449664 448 26.11607143 449 26.05790646 450 26.00000000 451 25.94235033 452 25.88495575 453 26.04856512 454 25.99118943 455 25.93406593 456 25.87719298 457 25.82056893 458 25.76419214 459 25.70806100 460 25.65217391 461 25.59652928 462 25.54112554 463 25.48596112 464 25.43103448 465 25.59139785 466 25.53648069 467 25.48179872 468 25.42735043 469 25.37313433 470 25.31914894 471 25.47770701 472 25.42372881 473 25.36997886 474 25.52742616 475 25.47368421 476 25.42016807 477 25.57651992 478 25.52301255 479 25.46972860 480 25.41666667 481 25.36382536 482 25.51867220 483 25.46583851 484 25.61983471 485 25.77319588 486 25.72016461 487 25.87268994 488 25.81967213 489 25.97137014 490 26.12244898 491 26.06924644 492 26.21951220 493 26.16632860 494 26.11336032 495 26.26262626 496 26.20967742 497 26.15694165 498 26.10441767 499 26.05210421 500 26.00000000 501 26.14770459 502 26.09561753 503 26.04373757 504 26.19047619 505 26.33663366 506 26.28458498 507 26.23274162 508 26.18110236 509 26.32612967 510 26.47058824 511 26.41878669 512 26.36718750 513 26.31578947 514 26.26459144 515 26.40776699 516 26.35658915 517 26.49903288 518 26.44787645 519 26.39691715 520 26.34615385 521 26.48752399 522 26.43678161 523 26.57743786 524 26.52671756 525 26.47619048 526 26.42585551 527 26.56546490 528 26.51515152 529 26.65406427 530 26.60377358 531 26.74199623 532 26.69172932 533 26.82926829 534 26.77902622 535 26.72897196 536 26.67910448 537 26.62942272 538 26.57992565 539 26.53061224 540 26.66666667 541 26.61737523 542 26.56826568 543 26.51933702 544 26.47058824 545 26.42201835 546 26.37362637 547 26.32541133 548 26.45985401 549 26.41165756 550 26.36363636 551 26.31578947 552 26.44927536 553 26.40144665 554 26.35379061 555 26.30630631 556 26.43884892 557 26.39138241 558 26.34408602 559 26.29695886 560 26.25000000 561 26.20320856 562 26.15658363 563 26.11012433 564 26.24113475 565 26.37168142 566 26.32508834 567 26.27865961 568 26.23239437 569 26.36203866 570 26.31578947 571 26.26970228 572 26.22377622 573 26.17801047 574 26.30662021 575 26.43478261 576 26.38888889 577 26.34315425 578 26.47058824 579 26.42487047 580 26.55172414 581 26.50602410 582 26.63230241 583 26.58662093 584 26.71232877 585 26.66666667 586 26.79180887 587 26.91652470 588 26.87074830 589 26.82512733 590 26.94915254 591 26.90355330 592 26.85810811 593 26.81281619 594 26.76767677 595 26.72268908 596 26.67785235 597 26.63316583 598 26.58862876 599 26.54424040 600 26.50000000 601 26.62229617 602 26.57807309 603 26.53399668 604 26.49006623 605 26.44628099 606 26.56765677 607 26.52388797 608 26.48026316 609 26.43678161 610 26.39344262 611 26.35024550 612 26.47058824 613 26.59053834 614 26.54723127 615 26.50406504 616 26.62337662 617 26.74230146 618 26.69902913 619 26.81744750 620 26.77419355 621 26.73107890 622 26.84887460 623 26.80577849 624 26.76282051 625 26.72000000 626 26.67731629 627 26.79425837 628 26.75159236 629 26.70906200 630 26.66666667 631 26.78288431 632 26.74050633 633 26.69826224 634 26.81388013 635 26.77165354 636 26.72955975 637 26.68759812 638 26.64576803 639 26.60406886 640 26.56250000 641 26.52106084 642 26.63551402 643 26.59409020 644 26.70807453 645 26.82170543 646 26.78018576 647 26.73879444 648 26.85185185 649 26.81047766 650 26.92307692 651 27.03533026 652 27.14723926 653 27.25880551 654 27.21712538 655 27.17557252 656 27.13414634 657 27.24505327 658 27.20364742 659 27.16236722 660 27.12121212 661 27.08018154 662 27.03927492 663 26.99849170 664 26.95783133 665 27.06766917 666 27.17717718 667 27.13643178 668 27.09580838 669 27.20478326 670 27.16417910 671 27.12369598 672 27.08333333 673 27.04309064 674 27.00296736 675 27.11111111 676 27.21893491 677 27.17872969 678 27.13864307 679 27.09867452 680 27.20588235 681 27.16593245 682 27.12609971 683 27.08638360 684 27.19298246 685 27.15328467 686 27.11370262 687 27.07423581 688 27.18023256 689 27.14078374 690 27.10144928 691 27.20694645 692 27.16763006 693 27.12842713 694 27.23342939 695 27.19424460 696 27.29885057 697 27.25968436 698 27.22063037 699 27.18168813 700 27.28571429 701 27.24679030 702 27.20797721 703 27.16927454 704 27.13068182 705 27.09219858 706 27.05382436 707 27.01555870 708 26.97740113 709 26.93935120 710 26.90140845 711 26.86357243 712 26.96629213 713 26.92847125 714 26.89075630 715 26.99300699 716 26.95530726 717 26.91771269 718 26.88022284 719 26.98191933 720 26.94444444 721 26.90707351 722 26.86980609 723 26.83264177 724 26.93370166 725 26.89655172 726 26.99724518 727 26.96011004 728 26.92307692 729 26.88614540 730 26.98630137 731 26.94938440 732 27.04918033 733 27.01227831 734 26.97547684 735 26.93877551 736 26.90217391 737 26.86567164 738 26.82926829 739 26.92828146 740 27.02702703 741 26.99055331 742 26.95417790 743 26.91790040 744 27.01612903 745 26.97986577 746 27.07774799 747 27.04149933 748 27.00534759 749 27.10280374 750 27.06666667 Final result: 27.0667 ±1.6235 Random chance: 19.8992 ±1.4588