common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 0.00000000 2 0.00000000 3 0.00000000 4 25.00000000 5 20.00000000 6 16.66666667 7 28.57142857 8 25.00000000 9 22.22222222 10 20.00000000 11 18.18181818 12 16.66666667 13 23.07692308 14 21.42857143 15 26.66666667 16 31.25000000 17 29.41176471 18 27.77777778 19 31.57894737 20 30.00000000 21 28.57142857 22 27.27272727 23 26.08695652 24 25.00000000 25 28.00000000 26 26.92307692 27 25.92592593 28 28.57142857 29 31.03448276 30 33.33333333 31 35.48387097 32 37.50000000 33 36.36363636 34 35.29411765 35 37.14285714 36 36.11111111 37 35.13513514 38 36.84210526 39 35.89743590 40 35.00000000 41 36.58536585 42 35.71428571 43 34.88372093 44 34.09090909 45 33.33333333 46 32.60869565 47 31.91489362 48 33.33333333 49 34.69387755 50 36.00000000 51 35.29411765 52 36.53846154 53 35.84905660 54 37.03703704 55 38.18181818 56 39.28571429 57 38.59649123 58 37.93103448 59 37.28813559 60 36.66666667 61 36.06557377 62 35.48387097 63 34.92063492 64 34.37500000 65 33.84615385 66 34.84848485 67 34.32835821 68 33.82352941 69 33.33333333 70 34.28571429 71 33.80281690 72 34.72222222 73 34.24657534 74 35.13513514 75 36.00000000 76 36.84210526 77 37.66233766 78 38.46153846 79 37.97468354 80 38.75000000 81 38.27160494 82 37.80487805 83 38.55421687 84 39.28571429 85 38.82352941 86 38.37209302 87 39.08045977 88 39.77272727 89 39.32584270 90 38.88888889 91 38.46153846 92 38.04347826 93 37.63440860 94 37.23404255 95 36.84210526 96 37.50000000 97 37.11340206 98 36.73469388 99 36.36363636 100 36.00000000 101 35.64356436 102 35.29411765 103 34.95145631 104 34.61538462 105 34.28571429 106 33.96226415 107 33.64485981 108 33.33333333 109 33.02752294 110 32.72727273 111 32.43243243 112 32.14285714 113 32.74336283 114 33.33333333 115 33.91304348 116 33.62068966 117 33.33333333 118 33.05084746 119 32.77310924 120 32.50000000 121 32.23140496 122 31.96721311 123 31.70731707 124 31.45161290 125 32.00000000 126 31.74603175 127 31.49606299 128 31.25000000 129 31.00775194 130 30.76923077 131 31.29770992 132 31.81818182 133 31.57894737 134 31.34328358 135 31.11111111 136 31.61764706 137 31.38686131 138 31.15942029 139 30.93525180 140 30.71428571 141 31.20567376 142 31.69014085 143 32.16783217 144 31.94444444 145 31.72413793 146 31.50684932 147 31.29251701 148 31.75675676 149 31.54362416 150 31.33333333 151 31.12582781 152 31.57894737 153 31.37254902 154 31.16883117 155 30.96774194 156 30.76923077 157 31.21019108 158 31.64556962 159 31.44654088 160 31.87500000 161 31.67701863 162 31.48148148 163 31.28834356 164 31.09756098 165 31.51515152 166 31.32530120 167 31.73652695 168 31.54761905 169 31.36094675 170 31.17647059 171 30.99415205 172 30.81395349 173 30.63583815 174 30.45977011 175 30.28571429 176 30.11363636 177 29.94350282 178 29.77528090 179 30.16759777 180 30.00000000 181 29.83425414 182 29.67032967 183 30.05464481 184 29.89130435 185 30.27027027 186 30.64516129 187 31.01604278 188 30.85106383 189 30.68783069 190 30.52631579 191 30.89005236 192 30.72916667 193 30.56994819 194 30.92783505 195 31.28205128 196 31.12244898 197 30.96446701 198 30.80808081 199 30.65326633 200 30.50000000 201 30.84577114 202 30.69306931 203 30.54187192 204 30.39215686 205 30.73170732 206 30.58252427 207 30.43478261 208 30.28846154 209 30.14354067 210 30.00000000 211 30.33175355 212 30.18867925 213 30.04694836 214 29.90654206 215 29.76744186 216 29.62962963 217 29.49308756 218 29.35779817 219 29.22374429 220 29.09090909 221 28.95927602 222 29.27927928 223 29.14798206 224 29.46428571 225 29.77777778 226 29.64601770 227 29.95594714 228 30.26315789 229 30.13100437 230 30.00000000 231 29.87012987 232 29.74137931 233 29.61373391 234 29.91452991 235 29.78723404 236 29.66101695 237 29.53586498 238 29.41176471 239 29.28870293 240 29.58333333 241 29.46058091 242 29.75206612 243 30.04115226 244 29.91803279 245 30.20408163 246 30.08130081 247 29.95951417 248 30.24193548 249 30.12048193 250 30.00000000 251 29.88047809 252 29.76190476 253 29.64426877 254 29.52755906 255 29.80392157 256 30.07812500 257 30.35019455 258 30.23255814 259 30.50193050 260 30.38461538 261 30.26819923 262 30.15267176 263 30.03802281 264 29.92424242 265 29.81132075 266 29.69924812 267 29.58801498 268 29.47761194 269 29.36802974 270 29.25925926 271 29.15129151 272 29.41176471 273 29.30402930 274 29.19708029 275 29.09090909 276 29.34782609 277 29.60288809 278 29.85611511 279 29.74910394 280 30.00000000 281 29.89323843 282 29.78723404 283 29.68197880 284 29.57746479 285 29.47368421 286 29.37062937 287 29.61672474 288 29.51388889 289 29.41176471 290 29.65517241 291 29.89690722 292 30.13698630 293 30.37542662 294 30.61224490 295 30.50847458 296 30.40540541 297 30.30303030 298 30.53691275 299 30.43478261 300 30.66666667 301 30.89700997 302 31.12582781 303 31.02310231 304 31.25000000 305 31.47540984 306 31.37254902 307 31.59609121 308 31.49350649 309 31.71521036 310 31.61290323 311 31.83279743 312 32.05128205 313 32.26837061 314 32.16560510 315 32.06349206 316 31.96202532 317 31.86119874 318 31.76100629 319 31.97492163 320 31.87500000 321 31.77570093 322 31.67701863 323 31.57894737 324 31.48148148 325 31.69230769 326 31.59509202 327 31.80428135 328 31.70731707 329 31.61094225 330 31.51515152 331 31.41993958 332 31.62650602 333 31.53153153 334 31.43712575 335 31.64179104 336 31.54761905 337 31.45400593 338 31.36094675 339 31.26843658 340 31.17647059 341 31.08504399 342 30.99415205 343 30.90379009 344 30.81395349 345 30.72463768 346 30.92485549 347 31.12391931 348 31.03448276 349 30.94555874 350 31.14285714 351 31.05413105 352 30.96590909 353 31.16147309 354 31.07344633 355 30.98591549 356 31.17977528 357 31.09243697 358 31.00558659 359 30.91922006 360 31.11111111 361 31.30193906 362 31.49171271 363 31.68044077 364 31.59340659 365 31.50684932 366 31.42076503 367 31.33514986 368 31.25000000 369 31.16531165 370 31.35135135 371 31.26684636 372 31.18279570 373 31.09919571 374 31.28342246 375 31.46666667 376 31.38297872 377 31.29973475 378 31.21693122 379 31.13456464 380 31.05263158 381 31.23359580 382 31.15183246 383 31.33159269 384 31.25000000 385 31.42857143 386 31.34715026 387 31.26614987 388 31.18556701 389 31.36246787 390 31.28205128 391 31.45780051 392 31.63265306 393 31.55216285 394 31.47208122 395 31.39240506 396 31.56565657 397 31.48614610 398 31.40703518 399 31.57894737 400 31.50000000 401 31.67082294 402 31.59203980 403 31.51364764 404 31.43564356 405 31.60493827 406 31.52709360 407 31.44963145 408 31.37254902 409 31.54034230 410 31.46341463 411 31.38686131 412 31.31067961 413 31.47699758 414 31.64251208 415 31.80722892 416 31.73076923 417 31.65467626 418 31.57894737 419 31.74224344 420 31.90476190 421 31.82897862 422 31.75355450 423 31.67848700 424 31.83962264 425 31.76470588 426 31.69014085 427 31.61592506 428 31.54205607 429 31.70163170 430 31.62790698 431 31.78654292 432 31.94444444 433 31.87066975 434 32.02764977 435 31.95402299 436 31.88073394 437 31.80778032 438 31.96347032 439 31.89066059 440 31.81818182 441 31.74603175 442 31.67420814 443 31.60270880 444 31.53153153 445 31.46067416 446 31.39013453 447 31.31991051 448 31.25000000 449 31.18040089 450 31.11111111 451 31.04212860 452 31.19469027 453 31.34657837 454 31.27753304 455 31.20879121 456 31.35964912 457 31.29102845 458 31.22270742 459 31.37254902 460 31.52173913 461 31.45336226 462 31.38528139 463 31.31749460 464 31.25000000 465 31.18279570 466 31.11587983 467 31.26338330 468 31.19658120 469 31.34328358 470 31.48936170 471 31.42250531 472 31.35593220 473 31.50105708 474 31.43459916 475 31.36842105 476 31.51260504 477 31.44654088 478 31.58995816 479 31.73277662 480 31.66666667 481 31.60083160 482 31.53526971 483 31.46997930 484 31.40495868 485 31.34020619 486 31.27572016 487 31.41683778 488 31.35245902 489 31.28834356 490 31.42857143 491 31.36456212 492 31.30081301 493 31.23732252 494 31.17408907 495 31.31313131 496 31.45161290 497 31.38832998 498 31.32530120 499 31.26252505 500 31.40000000 501 31.33732535 502 31.47410359 503 31.41153082 504 31.54761905 505 31.48514851 506 31.62055336 507 31.55818540 508 31.49606299 509 31.43418468 510 31.56862745 511 31.50684932 512 31.44531250 513 31.38401559 514 31.32295720 515 31.26213592 516 31.20155039 517 31.33462282 518 31.46718147 519 31.40655106 520 31.34615385 521 31.28598848 522 31.41762452 523 31.54875717 524 31.67938931 525 31.61904762 526 31.55893536 527 31.68880455 528 31.62878788 529 31.56899811 530 31.69811321 531 31.63841808 532 31.57894737 533 31.51969981 534 31.46067416 535 31.58878505 536 31.52985075 537 31.65735568 538 31.78438662 539 31.72541744 540 31.66666667 541 31.79297597 542 31.91881919 543 31.86003683 544 31.80147059 545 31.74311927 546 31.68498168 547 31.62705667 548 31.75182482 549 31.69398907 550 31.63636364 551 31.57894737 552 31.52173913 553 31.46473779 554 31.40794224 555 31.35135135 556 31.29496403 557 31.23877917 558 31.36200717 559 31.30590340 560 31.25000000 561 31.19429590 562 31.13879004 563 31.26110124 564 31.20567376 565 31.15044248 566 31.09540636 567 31.04056437 568 31.16197183 569 31.10720562 570 31.05263158 571 31.17338004 572 31.11888112 573 31.06457243 574 31.18466899 575 31.13043478 576 31.25000000 577 31.36915078 578 31.31487889 579 31.26079447 580 31.20689655 581 31.15318417 582 31.09965636 583 31.04631218 584 31.16438356 585 31.11111111 586 31.05802048 587 31.00511073 588 30.95238095 589 30.89983022 590 30.84745763 591 30.79526227 592 30.74324324 593 30.86003373 594 30.97643098 595 30.92436975 596 31.04026846 597 30.98827471 598 30.93645485 599 31.05175292 600 31.00000000 601 31.11480865 602 31.06312292 603 31.17744610 604 31.12582781 605 31.23966942 606 31.18811881 607 31.30148270 608 31.25000000 609 31.19868637 610 31.14754098 611 31.09656301 612 31.04575163 613 30.99510604 614 31.10749186 615 31.05691057 616 31.16883117 617 31.28038898 618 31.22977346 619 31.17932149 620 31.12903226 621 31.07890499 622 31.18971061 623 31.13964687 624 31.08974359 625 31.04000000 626 31.15015974 627 31.10047847 628 31.05095541 629 31.16057234 630 31.11111111 631 31.06180666 632 31.17088608 633 31.12164297 634 31.07255521 635 31.02362205 636 30.97484277 637 30.92621664 638 30.87774295 639 30.98591549 640 31.09375000 641 31.04524181 642 31.15264798 643 31.10419907 644 31.21118012 645 31.16279070 646 31.11455108 647 31.06646059 648 31.01851852 649 30.97072419 650 30.92307692 651 30.87557604 652 30.98159509 653 30.93415008 654 31.03975535 655 30.99236641 656 31.09756098 657 31.05022831 658 31.15501520 659 31.10773900 660 31.06060606 661 31.01361573 662 30.96676737 663 30.92006033 664 30.87349398 665 30.82706767 666 30.78078078 667 30.88455772 668 30.98802395 669 30.94170404 670 30.89552239 671 30.84947839 672 30.80357143 673 30.75780089 674 30.71216617 675 30.66666667 676 30.62130178 677 30.57607090 678 30.53097345 679 30.48600884 680 30.58823529 681 30.54331865 682 30.49853372 683 30.60029283 684 30.55555556 685 30.51094891 686 30.46647230 687 30.56768559 688 30.52325581 689 30.47895501 690 30.43478261 691 30.53545586 692 30.49132948 693 30.44733045 694 30.40345821 695 30.35971223 696 30.45977011 697 30.41606887 698 30.37249284 699 30.47210300 700 30.42857143 701 30.52781740 702 30.62678063 703 30.72546230 704 30.68181818 705 30.63829787 706 30.59490085 707 30.55162659 708 30.50847458 709 30.46544429 710 30.56338028 711 30.66104079 712 30.61797753 713 30.71528752 714 30.81232493 715 30.76923077 716 30.86592179 717 30.82287308 718 30.77994429 719 30.73713491 720 30.69444444 721 30.65187240 722 30.60941828 723 30.70539419 724 30.66298343 725 30.75862069 726 30.71625344 727 30.67400275 728 30.63186813 729 30.58984911 730 30.54794521 731 30.50615595 732 30.60109290 733 30.69577080 734 30.65395095 735 30.61224490 736 30.57065217 737 30.66485753 738 30.62330623 739 30.71718539 740 30.81081081 741 30.76923077 742 30.86253369 743 30.95558546 744 31.04838710 745 31.00671141 746 30.96514745 747 30.92369478 748 30.88235294 749 30.84112150 750 30.93333333 Final result: 30.9333 ±1.6889 Random chance: 25.0000 ±1.5822