common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 50.00000000 9 44.44444444 10 40.00000000 11 36.36363636 12 33.33333333 13 38.46153846 14 35.71428571 15 40.00000000 16 43.75000000 17 41.17647059 18 38.88888889 19 42.10526316 20 40.00000000 21 38.09523810 22 36.36363636 23 34.78260870 24 33.33333333 25 36.00000000 26 34.61538462 27 33.33333333 28 35.71428571 29 37.93103448 30 40.00000000 31 41.93548387 32 43.75000000 33 42.42424242 34 41.17647059 35 42.85714286 36 41.66666667 37 40.54054054 38 42.10526316 39 41.02564103 40 40.00000000 41 41.46341463 42 40.47619048 43 39.53488372 44 38.63636364 45 37.77777778 46 36.95652174 47 36.17021277 48 37.50000000 49 38.77551020 50 40.00000000 51 39.21568627 52 38.46153846 53 37.73584906 54 37.03703704 55 38.18181818 56 39.28571429 57 38.59649123 58 37.93103448 59 37.28813559 60 36.66666667 61 36.06557377 62 35.48387097 63 34.92063492 64 34.37500000 65 33.84615385 66 33.33333333 67 32.83582090 68 32.35294118 69 33.33333333 70 34.28571429 71 33.80281690 72 34.72222222 73 34.24657534 74 33.78378378 75 34.66666667 76 35.52631579 77 36.36363636 78 37.17948718 79 36.70886076 80 37.50000000 81 37.03703704 82 36.58536585 83 37.34939759 84 38.09523810 85 37.64705882 86 37.20930233 87 37.93103448 88 38.63636364 89 38.20224719 90 37.77777778 91 37.36263736 92 36.95652174 93 36.55913978 94 36.17021277 95 35.78947368 96 36.45833333 97 36.08247423 98 35.71428571 99 35.35353535 100 35.00000000 101 34.65346535 102 34.31372549 103 33.98058252 104 33.65384615 105 33.33333333 106 33.01886792 107 32.71028037 108 33.33333333 109 33.02752294 110 32.72727273 111 32.43243243 112 32.14285714 113 32.74336283 114 33.33333333 115 33.91304348 116 33.62068966 117 33.33333333 118 33.05084746 119 32.77310924 120 32.50000000 121 32.23140496 122 31.96721311 123 31.70731707 124 31.45161290 125 32.00000000 126 31.74603175 127 31.49606299 128 31.25000000 129 31.00775194 130 30.76923077 131 31.29770992 132 31.06060606 133 30.82706767 134 30.59701493 135 30.37037037 136 30.88235294 137 30.65693431 138 30.43478261 139 30.21582734 140 30.00000000 141 29.78723404 142 29.57746479 143 30.06993007 144 30.55555556 145 30.34482759 146 30.13698630 147 29.93197279 148 30.40540541 149 30.20134228 150 30.00000000 151 29.80132450 152 30.26315789 153 30.06535948 154 29.87012987 155 29.67741935 156 29.48717949 157 29.93630573 158 30.37974684 159 30.18867925 160 30.62500000 161 30.43478261 162 30.24691358 163 30.06134969 164 29.87804878 165 30.30303030 166 30.12048193 167 30.53892216 168 30.35714286 169 30.17751479 170 30.00000000 171 29.82456140 172 29.65116279 173 29.47976879 174 29.31034483 175 29.14285714 176 29.54545455 177 29.37853107 178 29.21348315 179 29.60893855 180 30.00000000 181 29.83425414 182 29.67032967 183 30.05464481 184 29.89130435 185 30.27027027 186 30.64516129 187 30.48128342 188 30.31914894 189 30.15873016 190 30.00000000 191 29.84293194 192 29.68750000 193 29.53367876 194 29.89690722 195 30.25641026 196 30.61224490 197 30.45685279 198 30.30303030 199 30.15075377 200 30.00000000 201 30.34825871 202 30.19801980 203 30.54187192 204 30.39215686 205 30.73170732 206 30.58252427 207 30.43478261 208 30.28846154 209 30.14354067 210 30.00000000 211 30.33175355 212 30.18867925 213 30.04694836 214 29.90654206 215 29.76744186 216 29.62962963 217 29.49308756 218 29.35779817 219 29.22374429 220 29.09090909 221 28.95927602 222 29.27927928 223 29.14798206 224 29.01785714 225 29.33333333 226 29.20353982 227 29.51541850 228 29.82456140 229 29.69432314 230 29.56521739 231 29.43722944 232 29.31034483 233 29.18454936 234 29.48717949 235 29.36170213 236 29.23728814 237 29.11392405 238 28.99159664 239 28.87029289 240 28.75000000 241 28.63070539 242 28.92561983 243 29.21810700 244 29.09836066 245 29.38775510 246 29.26829268 247 29.14979757 248 29.03225806 249 28.91566265 250 28.80000000 251 28.68525896 252 28.57142857 253 28.85375494 254 28.74015748 255 29.01960784 256 28.90625000 257 29.18287938 258 29.06976744 259 29.34362934 260 29.23076923 261 29.11877395 262 29.00763359 263 28.89733840 264 29.16666667 265 29.05660377 266 28.94736842 267 28.83895131 268 28.73134328 269 28.62453532 270 28.51851852 271 28.41328413 272 28.67647059 273 28.57142857 274 28.46715328 275 28.36363636 276 28.62318841 277 28.88086643 278 28.77697842 279 28.67383513 280 28.57142857 281 28.46975089 282 28.36879433 283 28.26855124 284 28.16901408 285 28.07017544 286 27.97202797 287 28.22299652 288 28.12500000 289 28.02768166 290 28.27586207 291 28.52233677 292 28.76712329 293 29.01023891 294 29.25170068 295 29.15254237 296 29.05405405 297 28.95622896 298 29.19463087 299 29.09698997 300 29.33333333 301 29.23588040 302 29.47019868 303 29.37293729 304 29.60526316 305 29.83606557 306 29.73856209 307 29.64169381 308 29.54545455 309 29.44983819 310 29.35483871 311 29.26045016 312 29.48717949 313 29.71246006 314 29.93630573 315 29.84126984 316 29.74683544 317 29.65299685 318 29.55974843 319 29.78056426 320 29.68750000 321 29.59501558 322 29.50310559 323 29.41176471 324 29.32098765 325 29.53846154 326 29.44785276 327 29.66360856 328 29.87804878 329 29.78723404 330 29.69696970 331 29.60725076 332 29.81927711 333 29.72972973 334 29.64071856 335 29.85074627 336 29.76190476 337 29.67359050 338 29.58579882 339 29.49852507 340 29.41176471 341 29.32551320 342 29.23976608 343 29.15451895 344 29.06976744 345 28.98550725 346 29.19075145 347 29.39481268 348 29.31034483 349 29.51289398 350 29.71428571 351 29.62962963 352 29.54545455 353 29.74504249 354 29.66101695 355 29.57746479 356 29.77528090 357 29.69187675 358 29.60893855 359 29.52646240 360 29.72222222 361 29.91689751 362 29.83425414 363 30.02754821 364 29.94505495 365 29.86301370 366 29.78142077 367 29.70027248 368 29.61956522 369 29.53929539 370 29.72972973 371 29.64959569 372 29.56989247 373 29.49061662 374 29.41176471 375 29.60000000 376 29.52127660 377 29.44297082 378 29.36507937 379 29.28759894 380 29.21052632 381 29.13385827 382 29.05759162 383 29.24281984 384 29.16666667 385 29.09090909 386 29.01554404 387 28.94056848 388 28.86597938 389 29.04884319 390 28.97435897 391 29.15601023 392 29.33673469 393 29.26208651 394 29.18781726 395 29.11392405 396 29.29292929 397 29.21914358 398 29.14572864 399 29.32330827 400 29.25000000 401 29.42643392 402 29.35323383 403 29.28039702 404 29.20792079 405 29.13580247 406 29.06403941 407 28.99262899 408 28.92156863 409 29.09535452 410 29.02439024 411 28.95377129 412 28.88349515 413 29.05569007 414 29.22705314 415 29.39759036 416 29.32692308 417 29.25659472 418 29.18660287 419 29.35560859 420 29.52380952 421 29.45368171 422 29.38388626 423 29.31442080 424 29.48113208 425 29.64705882 426 29.57746479 427 29.50819672 428 29.67289720 429 29.83682984 430 29.76744186 431 29.93039443 432 30.09259259 433 30.02309469 434 30.18433180 435 30.11494253 436 30.04587156 437 29.97711670 438 30.13698630 439 30.29612756 440 30.22727273 441 30.15873016 442 30.09049774 443 30.02257336 444 29.95495495 445 29.88764045 446 29.82062780 447 29.75391499 448 29.68750000 449 29.62138085 450 29.55555556 451 29.49002217 452 29.64601770 453 29.80132450 454 29.73568282 455 29.67032967 456 29.82456140 457 29.75929978 458 29.69432314 459 29.84749455 460 30.00000000 461 29.93492408 462 29.87012987 463 30.02159827 464 29.95689655 465 29.89247312 466 29.82832618 467 29.97858672 468 29.91452991 469 30.06396588 470 30.00000000 471 29.93630573 472 29.87288136 473 30.02114165 474 29.95780591 475 29.89473684 476 30.04201681 477 29.97903564 478 30.12552301 479 30.27139875 480 30.20833333 481 30.14553015 482 30.08298755 483 30.02070393 484 29.95867769 485 29.89690722 486 29.83539095 487 29.97946612 488 29.91803279 489 29.85685072 490 30.00000000 491 30.14256619 492 30.08130081 493 30.02028398 494 30.16194332 495 30.30303030 496 30.44354839 497 30.38229376 498 30.32128514 499 30.26052104 500 30.40000000 501 30.33932136 502 30.27888446 503 30.21868787 504 30.35714286 505 30.29702970 506 30.43478261 507 30.37475345 508 30.31496063 509 30.25540275 510 30.39215686 511 30.33268102 512 30.27343750 513 30.21442495 514 30.15564202 515 30.09708738 516 30.03875969 517 30.17408124 518 30.30888031 519 30.25048170 520 30.19230769 521 30.13435701 522 30.26819923 523 30.40152964 524 30.53435115 525 30.47619048 526 30.41825095 527 30.55028463 528 30.49242424 529 30.62381853 530 30.75471698 531 30.69679849 532 30.63909774 533 30.58161351 534 30.52434457 535 30.65420561 536 30.59701493 537 30.72625698 538 30.85501859 539 30.79777365 540 30.74074074 541 30.86876155 542 30.99630996 543 30.93922652 544 30.88235294 545 30.82568807 546 30.76923077 547 30.71297989 548 30.83941606 549 30.78324226 550 30.72727273 551 30.67150635 552 30.79710145 553 30.74141049 554 30.68592058 555 30.63063063 556 30.57553957 557 30.52064632 558 30.64516129 559 30.59033989 560 30.53571429 561 30.48128342 562 30.42704626 563 30.55062167 564 30.49645390 565 30.44247788 566 30.38869258 567 30.33509700 568 30.45774648 569 30.40421793 570 30.35087719 571 30.47285464 572 30.41958042 573 30.36649215 574 30.48780488 575 30.43478261 576 30.55555556 577 30.67590988 578 30.62283737 579 30.56994819 580 30.51724138 581 30.46471601 582 30.41237113 583 30.36020583 584 30.47945205 585 30.42735043 586 30.37542662 587 30.32367973 588 30.27210884 589 30.22071307 590 30.16949153 591 30.11844332 592 30.06756757 593 30.01686341 594 30.13468013 595 30.08403361 596 30.20134228 597 30.15075377 598 30.10033445 599 30.21702838 600 30.16666667 601 30.28286190 602 30.23255814 603 30.34825871 604 30.29801325 605 30.41322314 606 30.36303630 607 30.47775947 608 30.42763158 609 30.37766831 610 30.32786885 611 30.27823241 612 30.22875817 613 30.17944535 614 30.29315961 615 30.24390244 616 30.35714286 617 30.47001621 618 30.42071197 619 30.37156704 620 30.32258065 621 30.27375201 622 30.38585209 623 30.33707865 624 30.28846154 625 30.24000000 626 30.35143770 627 30.30303030 628 30.41401274 629 30.52464229 630 30.47619048 631 30.42789223 632 30.53797468 633 30.48973144 634 30.44164038 635 30.39370079 636 30.34591195 637 30.29827316 638 30.25078370 639 30.35993740 640 30.46875000 641 30.42121685 642 30.52959502 643 30.48211509 644 30.59006211 645 30.54263566 646 30.49535604 647 30.44822257 648 30.40123457 649 30.35439137 650 30.30769231 651 30.26113671 652 30.21472393 653 30.16845329 654 30.27522936 655 30.22900763 656 30.33536585 657 30.28919330 658 30.39513678 659 30.34901366 660 30.30303030 661 30.25718608 662 30.21148036 663 30.16591252 664 30.12048193 665 30.07518797 666 30.03003003 667 29.98500750 668 30.08982036 669 30.04484305 670 30.14925373 671 30.10432191 672 30.05952381 673 30.01485884 674 29.97032641 675 29.92592593 676 29.88165680 677 29.83751846 678 29.79351032 679 29.74963181 680 29.85294118 681 29.95594714 682 29.91202346 683 30.01464129 684 29.97076023 685 29.92700730 686 29.88338192 687 29.83988355 688 29.79651163 689 29.75326560 690 29.71014493 691 29.81186686 692 29.76878613 693 29.72582973 694 29.68299712 695 29.64028777 696 29.74137931 697 29.69870875 698 29.65616046 699 29.75679542 700 29.71428571 701 29.81455064 702 29.91452991 703 30.01422475 704 29.97159091 705 29.92907801 706 29.88668555 707 29.84441301 708 29.80225989 709 29.76022567 710 29.85915493 711 29.95780591 712 29.91573034 713 30.01402525 714 30.11204482 715 30.06993007 716 30.02793296 717 29.98605300 718 29.94428969 719 29.90264256 720 29.86111111 721 29.81969487 722 29.77839335 723 29.87551867 724 29.83425414 725 29.93103448 726 30.02754821 727 29.98624484 728 29.94505495 729 30.04115226 730 30.00000000 731 29.95896033 732 30.05464481 733 30.01364256 734 29.97275204 735 29.93197279 736 30.02717391 737 30.12211669 738 30.08130081 739 30.17591340 740 30.27027027 741 30.22941970 742 30.32345013 743 30.41722746 744 30.51075269 745 30.46979866 746 30.42895442 747 30.38821954 748 30.34759358 749 30.30707610 750 30.40000000 Final result: 30.4000 ±1.6807 Random chance: 25.0000 ±1.5822