common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 817 tasks in prompt multiple_choice_score: selecting 750 random tasks from 817 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 25.00000000 5 20.00000000 6 16.66666667 7 14.28571429 8 12.50000000 9 11.11111111 10 10.00000000 11 9.09090909 12 16.66666667 13 15.38461538 14 14.28571429 15 13.33333333 16 18.75000000 17 17.64705882 18 16.66666667 19 21.05263158 20 20.00000000 21 19.04761905 22 18.18181818 23 17.39130435 24 20.83333333 25 20.00000000 26 23.07692308 27 22.22222222 28 21.42857143 29 24.13793103 30 23.33333333 31 25.80645161 32 25.00000000 33 27.27272727 34 26.47058824 35 25.71428571 36 25.00000000 37 27.02702703 38 26.31578947 39 25.64102564 40 25.00000000 41 24.39024390 42 23.80952381 43 25.58139535 44 25.00000000 45 24.44444444 46 26.08695652 47 25.53191489 48 27.08333333 49 26.53061224 50 26.00000000 51 25.49019608 52 25.00000000 53 26.41509434 54 27.77777778 55 29.09090909 56 28.57142857 57 28.07017544 58 27.58620690 59 27.11864407 60 28.33333333 61 27.86885246 62 29.03225806 63 28.57142857 64 28.12500000 65 27.69230769 66 27.27272727 67 26.86567164 68 26.47058824 69 27.53623188 70 27.14285714 71 28.16901408 72 29.16666667 73 28.76712329 74 28.37837838 75 28.00000000 76 27.63157895 77 27.27272727 78 26.92307692 79 26.58227848 80 26.25000000 81 25.92592593 82 25.60975610 83 25.30120482 84 25.00000000 85 24.70588235 86 24.41860465 87 24.13793103 88 23.86363636 89 24.71910112 90 24.44444444 91 24.17582418 92 23.91304348 93 23.65591398 94 23.40425532 95 24.21052632 96 25.00000000 97 24.74226804 98 24.48979592 99 25.25252525 100 25.00000000 101 25.74257426 102 25.49019608 103 26.21359223 104 25.96153846 105 25.71428571 106 25.47169811 107 26.16822430 108 26.85185185 109 27.52293578 110 28.18181818 111 27.92792793 112 27.67857143 113 27.43362832 114 28.07017544 115 27.82608696 116 27.58620690 117 27.35042735 118 27.11864407 119 26.89075630 120 27.50000000 121 28.09917355 122 27.86885246 123 27.64227642 124 28.22580645 125 28.80000000 126 29.36507937 127 29.13385827 128 28.90625000 129 28.68217054 130 28.46153846 131 28.24427481 132 28.03030303 133 27.81954887 134 28.35820896 135 28.14814815 136 27.94117647 137 28.46715328 138 28.26086957 139 28.77697842 140 28.57142857 141 28.36879433 142 28.16901408 143 27.97202797 144 27.77777778 145 27.58620690 146 27.39726027 147 27.89115646 148 27.70270270 149 27.51677852 150 27.33333333 151 27.81456954 152 27.63157895 153 27.45098039 154 27.27272727 155 27.09677419 156 27.56410256 157 27.38853503 158 27.84810127 159 27.67295597 160 27.50000000 161 27.32919255 162 27.16049383 163 27.60736196 164 27.43902439 165 27.87878788 166 28.31325301 167 28.14371257 168 27.97619048 169 27.81065089 170 27.64705882 171 27.48538012 172 27.32558140 173 27.74566474 174 27.58620690 175 27.42857143 176 27.84090909 177 27.68361582 178 27.52808989 179 27.37430168 180 27.22222222 181 27.07182320 182 26.92307692 183 26.77595628 184 26.63043478 185 26.48648649 186 26.34408602 187 26.20320856 188 26.06382979 189 25.92592593 190 25.78947368 191 25.65445026 192 25.52083333 193 25.38860104 194 25.77319588 195 25.64102564 196 25.51020408 197 25.38071066 198 25.25252525 199 25.12562814 200 25.00000000 201 24.87562189 202 25.24752475 203 25.12315271 204 25.00000000 205 25.36585366 206 25.24271845 207 25.60386473 208 25.48076923 209 25.35885167 210 25.23809524 211 25.11848341 212 25.47169811 213 25.35211268 214 25.23364486 215 25.11627907 216 25.46296296 217 25.34562212 218 25.68807339 219 25.57077626 220 25.45454545 221 25.33936652 222 25.22522523 223 25.11210762 224 25.44642857 225 25.77777778 226 25.66371681 227 25.55066079 228 25.43859649 229 25.32751092 230 25.21739130 231 25.10822511 232 25.43103448 233 25.75107296 234 26.06837607 235 26.38297872 236 26.69491525 237 27.00421941 238 27.31092437 239 27.19665272 240 27.08333333 241 26.97095436 242 26.85950413 243 26.74897119 244 26.63934426 245 26.53061224 246 26.42276423 247 26.31578947 248 26.61290323 249 26.90763052 250 26.80000000 251 27.09163347 252 26.98412698 253 26.87747036 254 26.77165354 255 26.66666667 256 26.56250000 257 26.45914397 258 26.35658915 259 26.25482625 260 26.15384615 261 26.43678161 262 26.33587786 263 26.23574144 264 26.51515152 265 26.79245283 266 26.69172932 267 26.96629213 268 26.86567164 269 26.76579926 270 26.66666667 271 26.56826568 272 26.47058824 273 26.37362637 274 26.27737226 275 26.18181818 276 26.08695652 277 25.99277978 278 25.89928058 279 25.80645161 280 25.71428571 281 25.97864769 282 25.88652482 283 25.79505300 284 26.05633803 285 25.96491228 286 26.22377622 287 26.48083624 288 26.38888889 289 26.29757785 290 26.20689655 291 26.46048110 292 26.36986301 293 26.62116041 294 26.53061224 295 26.77966102 296 27.02702703 297 27.27272727 298 27.18120805 299 27.09030100 300 27.00000000 301 26.91029900 302 26.82119205 303 26.73267327 304 26.64473684 305 26.55737705 306 26.47058824 307 26.38436482 308 26.62337662 309 26.53721683 310 26.45161290 311 26.36655949 312 26.60256410 313 26.51757188 314 26.43312102 315 26.34920635 316 26.26582278 317 26.18296530 318 26.10062893 319 26.01880878 320 25.93750000 321 25.85669782 322 26.08695652 323 26.00619195 324 25.92592593 325 25.84615385 326 25.76687117 327 25.68807339 328 25.91463415 329 25.83586626 330 26.06060606 331 25.98187311 332 26.20481928 333 26.12612613 334 26.04790419 335 26.26865672 336 26.48809524 337 26.40949555 338 26.33136095 339 26.54867257 340 26.47058824 341 26.68621701 342 26.90058480 343 26.82215743 344 27.03488372 345 27.24637681 346 27.45664740 347 27.66570605 348 27.58620690 349 27.50716332 350 27.42857143 351 27.35042735 352 27.27272727 353 27.47875354 354 27.40112994 355 27.60563380 356 27.52808989 357 27.45098039 358 27.65363128 359 27.57660167 360 27.50000000 361 27.42382271 362 27.34806630 363 27.54820937 364 27.47252747 365 27.67123288 366 27.59562842 367 27.79291553 368 27.71739130 369 27.64227642 370 27.56756757 371 27.49326146 372 27.68817204 373 27.61394102 374 27.80748663 375 27.73333333 376 27.65957447 377 27.58620690 378 27.51322751 379 27.44063325 380 27.36842105 381 27.29658793 382 27.22513089 383 27.15404700 384 27.08333333 385 27.01298701 386 26.94300518 387 26.87338501 388 27.06185567 389 26.99228792 390 26.92307692 391 26.85421995 392 26.78571429 393 26.97201018 394 27.15736041 395 27.08860759 396 27.02020202 397 27.20403023 398 27.13567839 399 27.06766917 400 27.00000000 401 26.93266833 402 26.86567164 403 27.04714640 404 26.98019802 405 26.91358025 406 26.84729064 407 26.78132678 408 26.71568627 409 26.65036675 410 26.58536585 411 26.52068127 412 26.45631068 413 26.39225182 414 26.57004831 415 26.50602410 416 26.44230769 417 26.37889688 418 26.31578947 419 26.25298329 420 26.42857143 421 26.60332542 422 26.54028436 423 26.47754137 424 26.41509434 425 26.35294118 426 26.29107981 427 26.22950820 428 26.16822430 429 26.10722611 430 26.04651163 431 25.98607889 432 26.15740741 433 26.32794457 434 26.49769585 435 26.43678161 436 26.37614679 437 26.31578947 438 26.25570776 439 26.19589977 440 26.13636364 441 26.07709751 442 26.01809955 443 25.95936795 444 25.90090090 445 25.84269663 446 25.78475336 447 25.72706935 448 25.66964286 449 25.83518931 450 25.77777778 451 25.72062084 452 25.66371681 453 25.82781457 454 25.77092511 455 25.71428571 456 25.65789474 457 25.60175055 458 25.54585153 459 25.49019608 460 25.43478261 461 25.37960954 462 25.32467532 463 25.26997840 464 25.21551724 465 25.37634409 466 25.32188841 467 25.26766595 468 25.21367521 469 25.15991471 470 25.10638298 471 25.26539278 472 25.21186441 473 25.15856237 474 25.10548523 475 25.26315789 476 25.21008403 477 25.36687631 478 25.31380753 479 25.26096033 480 25.20833333 481 25.15592516 482 25.31120332 483 25.25879917 484 25.41322314 485 25.56701031 486 25.51440329 487 25.66735113 488 25.61475410 489 25.76687117 490 25.71428571 491 25.66191446 492 25.81300813 493 25.76064909 494 25.70850202 495 25.85858586 496 25.80645161 497 25.75452716 498 25.70281124 499 25.65130261 500 25.60000000 501 25.74850299 502 25.69721116 503 25.64612326 504 25.79365079 505 25.94059406 506 25.88932806 507 25.83826430 508 25.78740157 509 25.93320236 510 26.07843137 511 26.02739726 512 25.97656250 513 25.92592593 514 25.87548638 515 26.01941748 516 25.96899225 517 26.11218569 518 26.06177606 519 26.01156069 520 25.96153846 521 26.10364683 522 26.05363985 523 26.19502868 524 26.14503817 525 26.09523810 526 26.04562738 527 26.18595825 528 26.13636364 529 26.27599244 530 26.22641509 531 26.36534840 532 26.31578947 533 26.45403377 534 26.40449438 535 26.35514019 536 26.30597015 537 26.44320298 538 26.39405204 539 26.34508349 540 26.48148148 541 26.43253235 542 26.38376384 543 26.33517495 544 26.28676471 545 26.23853211 546 26.19047619 547 26.14259598 548 26.27737226 549 26.22950820 550 26.18181818 551 26.13430127 552 26.26811594 553 26.22061483 554 26.17328520 555 26.12612613 556 26.25899281 557 26.21184919 558 26.16487455 559 26.11806798 560 26.07142857 561 26.02495544 562 25.97864769 563 25.93250444 564 26.06382979 565 26.19469027 566 26.14840989 567 26.10229277 568 26.05633803 569 26.18629174 570 26.14035088 571 26.09457093 572 26.04895105 573 26.00349040 574 26.13240418 575 26.26086957 576 26.21527778 577 26.34315425 578 26.47058824 579 26.42487047 580 26.55172414 581 26.50602410 582 26.63230241 583 26.58662093 584 26.71232877 585 26.66666667 586 26.79180887 587 26.74616695 588 26.70068027 589 26.65534805 590 26.77966102 591 26.73434856 592 26.68918919 593 26.64418212 594 26.59932660 595 26.55462185 596 26.51006711 597 26.46566164 598 26.42140468 599 26.37729549 600 26.33333333 601 26.45590682 602 26.57807309 603 26.53399668 604 26.49006623 605 26.44628099 606 26.56765677 607 26.52388797 608 26.48026316 609 26.43678161 610 26.39344262 611 26.35024550 612 26.47058824 613 26.59053834 614 26.54723127 615 26.50406504 616 26.62337662 617 26.74230146 618 26.69902913 619 26.65589661 620 26.61290323 621 26.57004831 622 26.68810289 623 26.64526485 624 26.60256410 625 26.56000000 626 26.51757188 627 26.47527911 628 26.43312102 629 26.39109698 630 26.34920635 631 26.46592710 632 26.42405063 633 26.38230648 634 26.49842271 635 26.45669291 636 26.41509434 637 26.37362637 638 26.33228840 639 26.29107981 640 26.25000000 641 26.20904836 642 26.32398754 643 26.28304821 644 26.39751553 645 26.51162791 646 26.47058824 647 26.42967543 648 26.54320988 649 26.50231125 650 26.61538462 651 26.72811060 652 26.84049080 653 26.95252680 654 26.91131498 655 26.87022901 656 26.82926829 657 26.78843227 658 26.74772036 659 26.70713202 660 26.66666667 661 26.62632375 662 26.58610272 663 26.54600302 664 26.50602410 665 26.61654135 666 26.72672673 667 26.68665667 668 26.64670659 669 26.75635277 670 26.71641791 671 26.67660209 672 26.63690476 673 26.59732541 674 26.55786350 675 26.66666667 676 26.77514793 677 26.73559823 678 26.69616519 679 26.65684831 680 26.76470588 681 26.72540382 682 26.68621701 683 26.64714495 684 26.75438596 685 26.71532847 686 26.67638484 687 26.63755459 688 26.74418605 689 26.70537010 690 26.66666667 691 26.77279305 692 26.73410405 693 26.69552670 694 26.80115274 695 26.76258993 696 26.86781609 697 26.82926829 698 26.79083095 699 26.75250358 700 26.85714286 701 26.81883024 702 26.78062678 703 26.74253201 704 26.70454545 705 26.66666667 706 26.62889518 707 26.59123055 708 26.55367232 709 26.51622003 710 26.47887324 711 26.44163150 712 26.54494382 713 26.50771388 714 26.47058824 715 26.57342657 716 26.53631285 717 26.49930265 718 26.46239554 719 26.42559110 720 26.38888889 721 26.35228849 722 26.31578947 723 26.27939142 724 26.38121547 725 26.34482759 726 26.44628099 727 26.40990371 728 26.37362637 729 26.33744856 730 26.43835616 731 26.40218878 732 26.50273224 733 26.46657572 734 26.43051771 735 26.39455782 736 26.35869565 737 26.32293080 738 26.28726287 739 26.38700947 740 26.35135135 741 26.31578947 742 26.28032345 743 26.24495289 744 26.34408602 745 26.30872483 746 26.40750670 747 26.50602410 748 26.47058824 749 26.56875834 750 26.53333333 Final result: 26.5333 ±1.6132 Random chance: 19.8992 ±1.4588