common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 50.00000000 9 44.44444444 10 50.00000000 11 45.45454545 12 41.66666667 13 38.46153846 14 35.71428571 15 40.00000000 16 43.75000000 17 47.05882353 18 44.44444444 19 47.36842105 20 45.00000000 21 47.61904762 22 45.45454545 23 43.47826087 24 45.83333333 25 48.00000000 26 46.15384615 27 44.44444444 28 42.85714286 29 44.82758621 30 46.66666667 31 48.38709677 32 50.00000000 33 48.48484848 34 50.00000000 35 51.42857143 36 52.77777778 37 51.35135135 38 52.63157895 39 51.28205128 40 52.50000000 41 51.21951220 42 50.00000000 43 51.16279070 44 50.00000000 45 51.11111111 46 50.00000000 47 48.93617021 48 50.00000000 49 48.97959184 50 48.00000000 51 47.05882353 52 48.07692308 53 47.16981132 54 46.29629630 55 47.27272727 56 48.21428571 57 49.12280702 58 50.00000000 59 49.15254237 60 48.33333333 61 49.18032787 62 48.38709677 63 47.61904762 64 46.87500000 65 46.15384615 66 45.45454545 67 44.77611940 68 44.11764706 69 43.47826087 70 44.28571429 71 45.07042254 72 44.44444444 73 43.83561644 74 44.59459459 75 45.33333333 76 46.05263158 77 46.75324675 78 47.43589744 79 46.83544304 80 47.50000000 81 48.14814815 82 47.56097561 83 48.19277108 84 47.61904762 85 47.05882353 86 46.51162791 87 47.12643678 88 47.72727273 89 47.19101124 90 46.66666667 91 46.15384615 92 45.65217391 93 45.16129032 94 44.68085106 95 44.21052632 96 44.79166667 97 44.32989691 98 43.87755102 99 43.43434343 100 43.00000000 101 43.56435644 102 43.13725490 103 42.71844660 104 43.26923077 105 43.80952381 106 44.33962264 107 43.92523364 108 43.51851852 109 44.03669725 110 44.54545455 111 44.14414414 112 43.75000000 113 43.36283186 114 43.85964912 115 44.34782609 116 43.96551724 117 44.44444444 118 44.91525424 119 44.53781513 120 44.16666667 121 44.62809917 122 44.26229508 123 43.90243902 124 43.54838710 125 43.20000000 126 42.85714286 127 42.51968504 128 42.18750000 129 41.86046512 130 41.53846154 131 41.98473282 132 41.66666667 133 41.35338346 134 41.04477612 135 40.74074074 136 40.44117647 137 40.14598540 138 39.85507246 139 40.28776978 140 40.71428571 141 40.42553191 142 40.84507042 143 41.25874126 144 40.97222222 145 41.37931034 146 41.78082192 147 41.49659864 148 41.89189189 149 41.61073826 150 41.33333333 151 41.05960265 152 41.44736842 153 41.17647059 154 40.90909091 155 40.64516129 156 40.38461538 157 40.76433121 158 41.13924051 159 40.88050314 160 41.25000000 161 40.99378882 162 40.74074074 163 40.49079755 164 40.24390244 165 40.60606061 166 40.36144578 167 40.71856287 168 40.47619048 169 40.82840237 170 40.58823529 171 40.93567251 172 40.69767442 173 40.46242775 174 40.22988506 175 40.00000000 176 39.77272727 177 39.54802260 178 39.32584270 179 39.66480447 180 40.00000000 181 40.33149171 182 40.10989011 183 40.43715847 184 40.21739130 185 40.54054054 186 40.86021505 187 41.17647059 188 41.48936170 189 41.79894180 190 41.57894737 191 41.88481675 192 41.66666667 193 41.45077720 194 41.75257732 195 42.05128205 196 42.34693878 197 42.13197970 198 42.42424242 199 42.21105528 200 42.00000000 201 41.79104478 202 41.58415842 203 41.37931034 204 41.17647059 205 41.46341463 206 41.26213592 207 41.54589372 208 41.34615385 209 41.14832536 210 40.95238095 211 41.23222749 212 41.03773585 213 40.84507042 214 40.65420561 215 40.46511628 216 40.74074074 217 40.55299539 218 40.82568807 219 41.09589041 220 41.36363636 221 41.17647059 222 41.44144144 223 41.70403587 224 41.51785714 225 41.77777778 226 41.59292035 227 41.40969163 228 41.66666667 229 41.92139738 230 41.73913043 231 41.55844156 232 41.81034483 233 41.63090129 234 41.88034188 235 41.70212766 236 41.52542373 237 41.35021097 238 41.17647059 239 41.00418410 240 41.25000000 241 41.07883817 242 41.32231405 243 41.56378601 244 41.39344262 245 41.63265306 246 41.46341463 247 41.29554656 248 41.53225806 249 41.36546185 250 41.20000000 251 41.03585657 252 40.87301587 253 40.71146245 254 40.55118110 255 40.39215686 256 40.62500000 257 40.85603113 258 40.69767442 259 40.54054054 260 40.38461538 261 40.61302682 262 40.45801527 263 40.30418251 264 40.53030303 265 40.75471698 266 40.97744361 267 40.82397004 268 40.67164179 269 40.89219331 270 40.74074074 271 40.59040590 272 40.44117647 273 40.29304029 274 40.14598540 275 40.00000000 276 40.21739130 277 40.43321300 278 40.28776978 279 40.14336918 280 40.35714286 281 40.21352313 282 40.42553191 283 40.28268551 284 40.14084507 285 40.00000000 286 39.86013986 287 40.06968641 288 39.93055556 289 39.79238754 290 40.00000000 291 40.20618557 292 40.41095890 293 40.27303754 294 40.13605442 295 40.00000000 296 39.86486486 297 39.73063973 298 39.93288591 299 39.79933110 300 40.00000000 301 39.86710963 302 39.73509934 303 39.60396040 304 39.47368421 305 39.67213115 306 39.54248366 307 39.73941368 308 39.61038961 309 39.48220065 310 39.35483871 311 39.22829582 312 39.42307692 313 39.29712460 314 39.17197452 315 39.04761905 316 38.92405063 317 38.80126183 318 38.99371069 319 39.18495298 320 39.06250000 321 38.94080997 322 39.13043478 323 39.00928793 324 38.88888889 325 39.07692308 326 38.95705521 327 39.14373089 328 39.02439024 329 38.90577508 330 38.78787879 331 38.67069486 332 38.55421687 333 38.43843844 334 38.32335329 335 38.50746269 336 38.69047619 337 38.57566766 338 38.46153846 339 38.34808260 340 38.23529412 341 38.41642229 342 38.59649123 343 38.48396501 344 38.66279070 345 38.55072464 346 38.72832370 347 38.90489914 348 38.79310345 349 38.96848138 350 39.14285714 351 39.03133903 352 39.20454545 353 39.37677054 354 39.26553672 355 39.15492958 356 39.04494382 357 39.21568627 358 39.10614525 359 38.99721448 360 39.16666667 361 39.33518006 362 39.50276243 363 39.66942149 364 39.56043956 365 39.45205479 366 39.61748634 367 39.50953678 368 39.40217391 369 39.29539295 370 39.45945946 371 39.35309973 372 39.24731183 373 39.14209115 374 39.03743316 375 39.20000000 376 39.09574468 377 38.99204244 378 39.15343915 379 39.05013193 380 38.94736842 381 38.84514436 382 39.00523560 383 39.16449086 384 39.06250000 385 39.22077922 386 39.11917098 387 39.01808786 388 38.91752577 389 39.07455013 390 39.23076923 391 39.38618926 392 39.54081633 393 39.44020356 394 39.34010152 395 39.49367089 396 39.64646465 397 39.54659950 398 39.44723618 399 39.59899749 400 39.50000000 401 39.65087282 402 39.55223881 403 39.45409429 404 39.35643564 405 39.50617284 406 39.40886700 407 39.31203931 408 39.21568627 409 39.36430318 410 39.26829268 411 39.17274939 412 39.07766990 413 38.98305085 414 39.13043478 415 39.03614458 416 38.94230769 417 38.84892086 418 38.75598086 419 38.90214797 420 38.80952381 421 38.71733967 422 38.62559242 423 38.53427896 424 38.67924528 425 38.58823529 426 38.49765258 427 38.64168618 428 38.55140187 429 38.69463869 430 38.60465116 431 38.74709977 432 38.65740741 433 38.56812933 434 38.47926267 435 38.39080460 436 38.30275229 437 38.21510297 438 38.35616438 439 38.49658314 440 38.40909091 441 38.54875283 442 38.46153846 443 38.37471783 444 38.28828829 445 38.20224719 446 38.11659193 447 38.03131991 448 37.94642857 449 37.86191537 450 37.77777778 451 37.69401330 452 37.61061947 453 37.74834437 454 37.66519824 455 37.58241758 456 37.71929825 457 37.85557987 458 37.77292576 459 37.69063181 460 37.82608696 461 37.74403471 462 37.87878788 463 37.79697624 464 37.93103448 465 37.84946237 466 37.76824034 467 37.90149893 468 38.03418803 469 38.16631130 470 38.29787234 471 38.21656051 472 38.13559322 473 38.26638478 474 38.18565401 475 38.10526316 476 38.23529412 477 38.15513627 478 38.28451883 479 38.41336117 480 38.33333333 481 38.25363825 482 38.17427386 483 38.09523810 484 38.01652893 485 37.93814433 486 38.06584362 487 38.19301848 488 38.11475410 489 38.03680982 490 38.16326531 491 38.08553971 492 38.00813008 493 37.93103448 494 38.05668016 495 38.18181818 496 38.30645161 497 38.22937626 498 38.15261044 499 38.27655311 500 38.40000000 501 38.32335329 502 38.44621514 503 38.56858847 504 38.49206349 505 38.61386139 506 38.73517787 507 38.65877712 508 38.58267717 509 38.70333988 510 38.82352941 511 38.74755382 512 38.67187500 513 38.59649123 514 38.52140078 515 38.64077670 516 38.75968992 517 38.87814313 518 38.99613900 519 38.92100193 520 38.84615385 521 38.77159309 522 38.88888889 523 38.81453155 524 38.93129771 525 38.85714286 526 38.78326996 527 38.89943074 528 39.01515152 529 39.13043478 530 39.24528302 531 39.17137476 532 39.09774436 533 39.02439024 534 39.13857678 535 39.25233645 536 39.17910448 537 39.29236499 538 39.21933086 539 39.14656772 540 39.07407407 541 39.00184843 542 39.11439114 543 39.04235727 544 39.15441176 545 39.08256881 546 39.01098901 547 39.12248629 548 39.23357664 549 39.16211293 550 39.09090909 551 39.20145191 552 39.13043478 553 39.05967450 554 38.98916968 555 38.91891892 556 39.02877698 557 38.95870736 558 39.06810036 559 38.99821109 560 38.92857143 561 38.85918004 562 38.96797153 563 39.07637655 564 39.00709220 565 39.11504425 566 39.22261484 567 39.15343915 568 39.08450704 569 39.01581722 570 38.94736842 571 38.87915937 572 38.81118881 573 38.74345550 574 38.67595819 575 38.60869565 576 38.71527778 577 38.64818024 578 38.58131488 579 38.51468048 580 38.62068966 581 38.55421687 582 38.48797251 583 38.59348199 584 38.69863014 585 38.80341880 586 38.73720137 587 38.67120954 588 38.60544218 589 38.53989813 590 38.47457627 591 38.40947547 592 38.34459459 593 38.44856661 594 38.55218855 595 38.48739496 596 38.59060403 597 38.52596315 598 38.46153846 599 38.56427379 600 38.50000000 601 38.43594010 602 38.37209302 603 38.47429519 604 38.41059603 605 38.51239669 606 38.44884488 607 38.55024712 608 38.48684211 609 38.42364532 610 38.36065574 611 38.46153846 612 38.39869281 613 38.49918434 614 38.59934853 615 38.53658537 616 38.47402597 617 38.41166937 618 38.34951456 619 38.28756058 620 38.22580645 621 38.16425121 622 38.26366559 623 38.20224719 624 38.14102564 625 38.08000000 626 38.17891374 627 38.11802233 628 38.05732484 629 38.15580286 630 38.09523810 631 38.03486529 632 38.13291139 633 38.07266983 634 38.01261830 635 37.95275591 636 37.89308176 637 37.99058085 638 37.93103448 639 37.87167449 640 37.96875000 641 37.90951638 642 38.00623053 643 37.94712286 644 37.88819876 645 37.82945736 646 37.77089783 647 37.71251932 648 37.80864198 649 37.75038521 650 37.69230769 651 37.63440860 652 37.73006135 653 37.67228178 654 37.76758410 655 37.70992366 656 37.80487805 657 37.74733638 658 37.84194529 659 37.78452200 660 37.72727273 661 37.67019667 662 37.61329305 663 37.55656109 664 37.50000000 665 37.44360902 666 37.38738739 667 37.33133433 668 37.42514970 669 37.51868460 670 37.46268657 671 37.40685544 672 37.35119048 673 37.29569094 674 37.38872404 675 37.33333333 676 37.27810651 677 37.22304284 678 37.31563422 679 37.26067747 680 37.20588235 681 37.29809104 682 37.24340176 683 37.18887262 684 37.13450292 685 37.22627737 686 37.17201166 687 37.11790393 688 37.06395349 689 37.01015965 690 36.95652174 691 37.04775687 692 36.99421965 693 36.94083694 694 36.88760807 695 36.83453237 696 36.92528736 697 36.87230990 698 36.96275072 699 37.05293276 700 37.00000000 701 36.94721826 702 37.03703704 703 37.12660028 704 37.21590909 705 37.16312057 706 37.11048159 707 37.05799151 708 37.00564972 709 36.95345557 710 36.90140845 711 36.99015471 712 36.93820225 713 37.02664797 714 37.11484594 715 37.06293706 716 37.15083799 717 37.09902371 718 37.04735376 719 36.99582754 720 36.94444444 721 36.89320388 722 36.84210526 723 36.92946058 724 36.87845304 725 36.96551724 726 36.91460055 727 37.00137552 728 37.08791209 729 37.03703704 730 36.98630137 731 36.93570451 732 36.88524590 733 36.97135061 734 36.92098093 735 36.87074830 736 36.82065217 737 36.90637720 738 36.99186992 739 37.07713126 740 37.02702703 741 36.97705803 742 37.06199461 743 37.01211306 744 37.09677419 745 37.04697987 746 36.99731903 747 36.94779116 748 36.89839572 749 36.84913218 750 36.93333333 Final result: 36.9333 +/- 1.7635 Random chance: 25.0000 +/- 1.5822