common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 869 tasks in prompt multiple_choice_score: selecting 750 random tasks from 869 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 66.66666667 4 50.00000000 5 60.00000000 6 66.66666667 7 57.14285714 8 62.50000000 9 55.55555556 10 50.00000000 11 45.45454545 12 50.00000000 13 46.15384615 14 50.00000000 15 53.33333333 16 50.00000000 17 52.94117647 18 50.00000000 19 47.36842105 20 45.00000000 21 42.85714286 22 45.45454545 23 43.47826087 24 45.83333333 25 44.00000000 26 42.30769231 27 44.44444444 28 42.85714286 29 41.37931034 30 43.33333333 31 41.93548387 32 40.62500000 33 42.42424242 34 44.11764706 35 42.85714286 36 41.66666667 37 43.24324324 38 44.73684211 39 43.58974359 40 45.00000000 41 46.34146341 42 45.23809524 43 46.51162791 44 45.45454545 45 44.44444444 46 45.65217391 47 46.80851064 48 47.91666667 49 46.93877551 50 48.00000000 51 47.05882353 52 48.07692308 53 49.05660377 54 48.14814815 55 47.27272727 56 48.21428571 57 49.12280702 58 50.00000000 59 49.15254237 60 50.00000000 61 50.81967213 62 50.00000000 63 50.79365079 64 50.00000000 65 49.23076923 66 50.00000000 67 49.25373134 68 50.00000000 69 50.72463768 70 50.00000000 71 49.29577465 72 48.61111111 73 49.31506849 74 48.64864865 75 49.33333333 76 50.00000000 77 50.64935065 78 51.28205128 79 50.63291139 80 51.25000000 81 50.61728395 82 51.21951220 83 50.60240964 84 51.19047619 85 51.76470588 86 52.32558140 87 52.87356322 88 53.40909091 89 53.93258427 90 54.44444444 91 54.94505495 92 55.43478261 93 54.83870968 94 54.25531915 95 54.73684211 96 54.16666667 97 53.60824742 98 54.08163265 99 54.54545455 100 55.00000000 101 54.45544554 102 54.90196078 103 54.36893204 104 54.80769231 105 55.23809524 106 54.71698113 107 55.14018692 108 54.62962963 109 54.12844037 110 54.54545455 111 54.95495495 112 55.35714286 113 54.86725664 114 54.38596491 115 54.78260870 116 54.31034483 117 54.70085470 118 55.08474576 119 54.62184874 120 55.00000000 121 54.54545455 122 54.91803279 123 55.28455285 124 54.83870968 125 55.20000000 126 55.55555556 127 55.90551181 128 55.46875000 129 55.03875969 130 54.61538462 131 54.96183206 132 55.30303030 133 55.63909774 134 55.22388060 135 54.81481481 136 54.41176471 137 54.74452555 138 55.07246377 139 55.39568345 140 55.00000000 141 55.31914894 142 54.92957746 143 55.24475524 144 54.86111111 145 55.17241379 146 54.79452055 147 55.10204082 148 54.72972973 149 55.03355705 150 55.33333333 151 55.62913907 152 55.92105263 153 56.20915033 154 55.84415584 155 55.48387097 156 55.76923077 157 56.05095541 158 55.69620253 159 55.97484277 160 55.62500000 161 55.90062112 162 56.17283951 163 55.82822086 164 55.48780488 165 55.75757576 166 56.02409639 167 55.68862275 168 55.95238095 169 55.62130178 170 55.29411765 171 55.55555556 172 55.23255814 173 55.49132948 174 55.74712644 175 55.42857143 176 55.11363636 177 55.36723164 178 55.05617978 179 55.30726257 180 55.00000000 181 55.24861878 182 55.49450549 183 55.19125683 184 54.89130435 185 55.13513514 186 55.37634409 187 55.08021390 188 54.78723404 189 54.49735450 190 54.21052632 191 54.45026178 192 54.16666667 193 53.88601036 194 54.12371134 195 54.35897436 196 54.59183673 197 54.82233503 198 55.05050505 199 55.27638191 200 55.50000000 201 55.72139303 202 55.94059406 203 55.66502463 204 55.88235294 205 55.60975610 206 55.33980583 207 55.55555556 208 55.76923077 209 55.98086124 210 56.19047619 211 56.39810427 212 56.13207547 213 55.86854460 214 56.07476636 215 56.27906977 216 56.01851852 217 55.76036866 218 55.96330275 219 56.16438356 220 55.90909091 221 55.65610860 222 55.40540541 223 55.15695067 224 55.35714286 225 55.11111111 226 55.30973451 227 55.50660793 228 55.70175439 229 55.89519651 230 56.08695652 231 55.84415584 232 55.60344828 233 55.36480687 234 55.12820513 235 54.89361702 236 54.66101695 237 54.43037975 238 54.62184874 239 54.81171548 240 55.00000000 241 55.18672199 242 54.95867769 243 54.73251029 244 54.50819672 245 54.28571429 246 54.06504065 247 54.25101215 248 54.43548387 249 54.21686747 250 54.40000000 251 54.58167331 252 54.76190476 253 54.54545455 254 54.33070866 255 54.50980392 256 54.68750000 257 54.86381323 258 54.65116279 259 54.44015444 260 54.61538462 261 54.78927203 262 54.96183206 263 55.13307985 264 55.30303030 265 55.09433962 266 54.88721805 267 55.05617978 268 54.85074627 269 55.01858736 270 55.18518519 271 54.98154982 272 54.77941176 273 54.94505495 274 54.74452555 275 54.54545455 276 54.71014493 277 54.87364621 278 54.67625899 279 54.83870968 280 54.64285714 281 54.80427046 282 54.96453901 283 54.77031802 284 54.92957746 285 54.73684211 286 54.89510490 287 55.05226481 288 55.20833333 289 55.01730104 290 55.17241379 291 55.32646048 292 55.13698630 293 55.29010239 294 55.10204082 295 55.25423729 296 55.06756757 297 55.21885522 298 55.03355705 299 55.18394649 300 55.00000000 301 54.81727575 302 54.96688742 303 54.78547855 304 54.93421053 305 55.08196721 306 54.90196078 307 55.04885993 308 54.87012987 309 55.01618123 310 55.16129032 311 55.30546624 312 55.12820513 313 55.27156550 314 55.09554140 315 54.92063492 316 55.06329114 317 55.20504732 318 55.03144654 319 54.85893417 320 54.68750000 321 54.51713396 322 54.65838509 323 54.48916409 324 54.32098765 325 54.15384615 326 54.29447853 327 54.12844037 328 53.96341463 329 54.10334347 330 54.24242424 331 54.38066465 332 54.21686747 333 54.35435435 334 54.19161677 335 54.02985075 336 53.86904762 337 53.70919881 338 53.55029586 339 53.68731563 340 53.82352941 341 53.95894428 342 53.80116959 343 53.93586006 344 54.06976744 345 53.91304348 346 54.04624277 347 53.89048991 348 53.73563218 349 53.86819484 350 53.71428571 351 53.84615385 352 53.69318182 353 53.54107649 354 53.38983051 355 53.23943662 356 53.08988764 357 52.94117647 358 53.07262570 359 52.92479109 360 53.05555556 361 53.18559557 362 53.31491713 363 53.44352617 364 53.29670330 365 53.15068493 366 53.00546448 367 52.86103542 368 52.98913043 369 52.84552846 370 52.70270270 371 52.56064690 372 52.41935484 373 52.27882038 374 52.40641711 375 52.53333333 376 52.65957447 377 52.78514589 378 52.91005291 379 52.77044855 380 52.89473684 381 52.75590551 382 52.61780105 383 52.48041775 384 52.34375000 385 52.46753247 386 52.59067358 387 52.45478036 388 52.31958763 389 52.18508997 390 52.30769231 391 52.17391304 392 52.04081633 393 52.16284987 394 52.28426396 395 52.40506329 396 52.27272727 397 52.39294710 398 52.26130653 399 52.13032581 400 52.25000000 401 52.11970075 402 52.23880597 403 52.35732010 404 52.47524752 405 52.59259259 406 52.70935961 407 52.82555283 408 52.94117647 409 52.81173594 410 52.92682927 411 53.04136253 412 53.15533981 413 53.02663438 414 52.89855072 415 52.77108434 416 52.64423077 417 52.51798561 418 52.39234450 419 52.26730310 420 52.38095238 421 52.25653207 422 52.36966825 423 52.48226950 424 52.35849057 425 52.23529412 426 52.11267606 427 51.99063232 428 51.86915888 429 51.98135198 430 52.09302326 431 52.20417633 432 52.08333333 433 51.96304850 434 51.84331797 435 51.95402299 436 52.06422018 437 52.17391304 438 52.05479452 439 51.93621868 440 52.04545455 441 52.15419501 442 52.03619910 443 51.91873589 444 51.80180180 445 51.91011236 446 52.01793722 447 52.12527964 448 52.23214286 449 52.33853007 450 52.44444444 451 52.32815965 452 52.43362832 453 52.53863135 454 52.42290749 455 52.30769231 456 52.19298246 457 52.07877462 458 51.96506550 459 52.06971678 460 52.17391304 461 52.06073753 462 52.16450216 463 52.05183585 464 51.93965517 465 52.04301075 466 52.14592275 467 52.03426124 468 51.92307692 469 51.81236674 470 51.91489362 471 52.01698514 472 52.11864407 473 52.00845666 474 51.89873418 475 51.78947368 476 51.68067227 477 51.78197065 478 51.88284519 479 51.98329854 480 51.87500000 481 51.76715177 482 51.86721992 483 51.96687371 484 52.06611570 485 52.16494845 486 52.26337449 487 52.36139630 488 52.25409836 489 52.14723926 490 52.04081633 491 52.13849287 492 52.03252033 493 51.92697769 494 52.02429150 495 51.91919192 496 51.81451613 497 51.91146881 498 52.00803213 499 52.10420842 500 52.20000000 501 52.09580838 502 52.19123506 503 52.28628231 504 52.18253968 505 52.27722772 506 52.17391304 507 52.07100592 508 52.16535433 509 52.06286837 510 52.15686275 511 52.05479452 512 52.14843750 513 52.04678363 514 52.14007782 515 52.23300971 516 52.13178295 517 52.03094778 518 51.93050193 519 51.83044316 520 51.73076923 521 51.82341651 522 51.91570881 523 51.81644359 524 51.71755725 525 51.61904762 526 51.52091255 527 51.42314991 528 51.32575758 529 51.22873346 530 51.32075472 531 51.41242938 532 51.50375940 533 51.59474672 534 51.49812734 535 51.40186916 536 51.30597015 537 51.39664804 538 51.30111524 539 51.39146568 540 51.29629630 541 51.20147874 542 51.29151292 543 51.38121547 544 51.28676471 545 51.37614679 546 51.28205128 547 51.18829982 548 51.27737226 549 51.36612022 550 51.45454545 551 51.36116152 552 51.26811594 553 51.35623870 554 51.44404332 555 51.53153153 556 51.43884892 557 51.52603232 558 51.43369176 559 51.52057245 560 51.60714286 561 51.69340463 562 51.77935943 563 51.86500888 564 51.77304965 565 51.68141593 566 51.76678445 567 51.85185185 568 51.93661972 569 51.84534271 570 51.75438596 571 51.66374781 572 51.57342657 573 51.65794066 574 51.74216028 575 51.82608696 576 51.90972222 577 51.99306759 578 51.90311419 579 51.98618307 580 52.06896552 581 51.97934596 582 51.89003436 583 51.97255575 584 52.05479452 585 52.13675214 586 52.21843003 587 52.12947189 588 52.04081633 589 52.12224109 590 52.20338983 591 52.28426396 592 52.36486486 593 52.27655987 594 52.35690236 595 52.43697479 596 52.34899329 597 52.26130653 598 52.17391304 599 52.25375626 600 52.16666667 601 52.07986689 602 51.99335548 603 51.90713101 604 51.82119205 605 51.73553719 606 51.81518152 607 51.72981878 608 51.80921053 609 51.72413793 610 51.80327869 611 51.71849427 612 51.63398693 613 51.54975530 614 51.46579805 615 51.54471545 616 51.62337662 617 51.70178282 618 51.61812298 619 51.53473344 620 51.61290323 621 51.52979066 622 51.44694534 623 51.52487961 624 51.44230769 625 51.36000000 626 51.27795527 627 51.35566188 628 51.27388535 629 51.19236884 630 51.26984127 631 51.18858954 632 51.10759494 633 51.02685624 634 50.94637224 635 50.86614173 636 50.94339623 637 50.86342229 638 50.94043887 639 50.86071987 640 50.78125000 641 50.85803432 642 50.77881620 643 50.69984448 644 50.77639752 645 50.69767442 646 50.61919505 647 50.54095827 648 50.46296296 649 50.38520801 650 50.30769231 651 50.38402458 652 50.30674847 653 50.22970904 654 50.30581040 655 50.22900763 656 50.15243902 657 50.07610350 658 50.15197568 659 50.07587253 660 50.15151515 661 50.07564297 662 50.15105740 663 50.22624434 664 50.30120482 665 50.37593985 666 50.45045045 667 50.52473763 668 50.44910180 669 50.37369208 670 50.44776119 671 50.37257824 672 50.29761905 673 50.22288262 674 50.14836795 675 50.07407407 676 50.00000000 677 50.07385524 678 50.14749263 679 50.22091311 680 50.29411765 681 50.22026432 682 50.14662757 683 50.21961933 684 50.14619883 685 50.21897810 686 50.14577259 687 50.07278020 688 50.14534884 689 50.21770682 690 50.28985507 691 50.36179450 692 50.43352601 693 50.50505051 694 50.57636888 695 50.50359712 696 50.43103448 697 50.50215208 698 50.42979943 699 50.50071531 700 50.57142857 701 50.64194009 702 50.71225071 703 50.64011380 704 50.71022727 705 50.78014184 706 50.84985836 707 50.91937765 708 50.98870056 709 51.05782793 710 51.12676056 711 51.19549930 712 51.26404494 713 51.33239832 714 51.26050420 715 51.18881119 716 51.25698324 717 51.18549512 718 51.11420613 719 51.04311544 720 51.11111111 721 51.04022191 722 51.10803324 723 51.17565698 724 51.24309392 725 51.31034483 726 51.23966942 727 51.16918845 728 51.23626374 729 51.30315501 730 51.36986301 731 51.29958960 732 51.36612022 733 51.43246930 734 51.49863760 735 51.42857143 736 51.49456522 737 51.42469471 738 51.35501355 739 51.42083897 740 51.35135135 741 51.41700405 742 51.48247978 743 51.54777927 744 51.47849462 745 51.54362416 746 51.47453083 747 51.40562249 748 51.33689840 749 51.26835781 750 51.33333333 Final result: 51.3333 ±1.8263 Random chance: 25.0083 ±1.5824