common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 1548 tasks in prompt multiple_choice_score: selecting 750 random tasks from 1548 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 50.00000000 5 40.00000000 6 33.33333333 7 42.85714286 8 50.00000000 9 44.44444444 10 40.00000000 11 36.36363636 12 33.33333333 13 38.46153846 14 35.71428571 15 40.00000000 16 43.75000000 17 41.17647059 18 38.88888889 19 42.10526316 20 40.00000000 21 38.09523810 22 36.36363636 23 34.78260870 24 33.33333333 25 36.00000000 26 34.61538462 27 33.33333333 28 35.71428571 29 37.93103448 30 40.00000000 31 38.70967742 32 40.62500000 33 39.39393939 34 38.23529412 35 40.00000000 36 38.88888889 37 37.83783784 38 36.84210526 39 35.89743590 40 35.00000000 41 34.14634146 42 33.33333333 43 32.55813953 44 31.81818182 45 31.11111111 46 30.43478261 47 29.78723404 48 31.25000000 49 32.65306122 50 34.00000000 51 33.33333333 52 32.69230769 53 32.07547170 54 31.48148148 55 32.72727273 56 33.92857143 57 35.08771930 58 34.48275862 59 33.89830508 60 33.33333333 61 32.78688525 62 32.25806452 63 31.74603175 64 31.25000000 65 32.30769231 66 33.33333333 67 32.83582090 68 32.35294118 69 31.88405797 70 32.85714286 71 32.39436620 72 33.33333333 73 32.87671233 74 33.78378378 75 34.66666667 76 35.52631579 77 36.36363636 78 37.17948718 79 36.70886076 80 36.25000000 81 37.03703704 82 36.58536585 83 37.34939759 84 38.09523810 85 37.64705882 86 37.20930233 87 37.93103448 88 38.63636364 89 38.20224719 90 37.77777778 91 37.36263736 92 36.95652174 93 36.55913978 94 37.23404255 95 36.84210526 96 37.50000000 97 37.11340206 98 36.73469388 99 36.36363636 100 37.00000000 101 36.63366337 102 36.27450980 103 35.92233010 104 35.57692308 105 36.19047619 106 35.84905660 107 35.51401869 108 35.18518519 109 34.86238532 110 34.54545455 111 34.23423423 112 33.92857143 113 33.62831858 114 34.21052632 115 34.78260870 116 34.48275862 117 34.18803419 118 33.89830508 119 33.61344538 120 33.33333333 121 33.05785124 122 32.78688525 123 32.52032520 124 32.25806452 125 32.80000000 126 33.33333333 127 33.07086614 128 32.81250000 129 32.55813953 130 32.30769231 131 32.82442748 132 33.33333333 133 33.08270677 134 32.83582090 135 32.59259259 136 32.35294118 137 32.11678832 138 31.88405797 139 31.65467626 140 31.42857143 141 31.91489362 142 31.69014085 143 32.16783217 144 32.63888889 145 32.41379310 146 32.19178082 147 31.97278912 148 32.43243243 149 32.21476510 150 32.00000000 151 31.78807947 152 32.23684211 153 32.02614379 154 31.81818182 155 31.61290323 156 31.41025641 157 31.84713376 158 31.64556962 159 31.44654088 160 31.87500000 161 31.67701863 162 31.48148148 163 31.28834356 164 31.70731707 165 32.12121212 166 31.92771084 167 32.33532934 168 32.14285714 169 31.95266272 170 31.76470588 171 31.57894737 172 31.39534884 173 31.21387283 174 31.03448276 175 30.85714286 176 30.68181818 177 30.50847458 178 30.33707865 179 30.72625698 180 30.55555556 181 30.38674033 182 30.21978022 183 30.60109290 184 30.43478261 185 30.27027027 186 30.64516129 187 30.48128342 188 30.31914894 189 30.15873016 190 30.00000000 191 30.36649215 192 30.20833333 193 30.05181347 194 30.41237113 195 30.76923077 196 30.61224490 197 30.45685279 198 30.30303030 199 30.15075377 200 30.00000000 201 30.34825871 202 30.19801980 203 30.04926108 204 29.90196078 205 30.24390244 206 30.09708738 207 29.95169082 208 29.80769231 209 29.66507177 210 29.52380952 211 29.85781991 212 29.71698113 213 29.57746479 214 29.43925234 215 29.30232558 216 29.62962963 217 29.49308756 218 29.35779817 219 29.22374429 220 29.09090909 221 28.95927602 222 29.27927928 223 29.14798206 224 29.01785714 225 29.33333333 226 29.20353982 227 29.51541850 228 29.82456140 229 29.69432314 230 29.56521739 231 29.43722944 232 29.31034483 233 29.18454936 234 29.48717949 235 29.36170213 236 29.23728814 237 29.11392405 238 28.99159664 239 28.87029289 240 29.16666667 241 29.04564315 242 29.33884298 243 29.62962963 244 29.50819672 245 29.79591837 246 29.67479675 247 29.55465587 248 29.83870968 249 29.71887550 250 29.60000000 251 29.48207171 252 29.36507937 253 29.24901186 254 29.13385827 255 29.41176471 256 29.29687500 257 29.57198444 258 29.45736434 259 29.34362934 260 29.23076923 261 29.11877395 262 29.00763359 263 28.89733840 264 28.78787879 265 28.67924528 266 28.94736842 267 28.83895131 268 28.73134328 269 28.62453532 270 28.51851852 271 28.41328413 272 28.67647059 273 28.57142857 274 28.46715328 275 28.36363636 276 28.62318841 277 28.88086643 278 29.13669065 279 29.03225806 280 29.28571429 281 29.18149466 282 29.07801418 283 28.97526502 284 28.87323944 285 28.77192982 286 28.67132867 287 28.91986063 288 28.81944444 289 28.71972318 290 28.96551724 291 29.20962199 292 29.45205479 293 29.69283276 294 29.93197279 295 29.83050847 296 29.72972973 297 29.62962963 298 29.86577181 299 29.76588629 300 30.00000000 301 30.23255814 302 30.46357616 303 30.36303630 304 30.59210526 305 30.81967213 306 30.71895425 307 30.61889251 308 30.51948052 309 30.42071197 310 30.32258065 311 30.54662379 312 30.76923077 313 30.99041534 314 31.21019108 315 31.11111111 316 31.01265823 317 30.91482650 318 30.81761006 319 31.03448276 320 30.93750000 321 30.84112150 322 30.74534161 323 30.65015480 324 30.55555556 325 30.76923077 326 30.67484663 327 30.88685015 328 31.09756098 329 31.00303951 330 30.90909091 331 30.81570997 332 31.02409639 333 30.93093093 334 30.83832335 335 31.04477612 336 30.95238095 337 30.86053412 338 30.76923077 339 30.67846608 340 30.88235294 341 30.79178886 342 30.70175439 343 30.90379009 344 30.81395349 345 30.72463768 346 30.92485549 347 31.12391931 348 31.03448276 349 30.94555874 350 31.14285714 351 31.05413105 352 30.96590909 353 31.16147309 354 31.07344633 355 30.98591549 356 31.17977528 357 31.09243697 358 31.00558659 359 30.91922006 360 31.11111111 361 31.30193906 362 31.49171271 363 31.68044077 364 31.59340659 365 31.50684932 366 31.69398907 367 31.60762943 368 31.52173913 369 31.43631436 370 31.62162162 371 31.53638814 372 31.45161290 373 31.36729223 374 31.55080214 375 31.73333333 376 31.64893617 377 31.56498674 378 31.48148148 379 31.39841689 380 31.31578947 381 31.49606299 382 31.67539267 383 31.85378590 384 31.77083333 385 31.68831169 386 31.60621762 387 31.52454780 388 31.44329897 389 31.61953728 390 31.53846154 391 31.71355499 392 31.88775510 393 31.80661578 394 31.72588832 395 31.64556962 396 31.81818182 397 31.73803526 398 31.65829146 399 31.82957393 400 31.75000000 401 31.92019950 402 32.08955224 403 32.00992556 404 31.93069307 405 31.85185185 406 31.77339901 407 31.69533170 408 31.61764706 409 31.78484108 410 31.70731707 411 31.63017032 412 31.55339806 413 31.71912833 414 31.88405797 415 32.04819277 416 32.21153846 417 32.13429257 418 32.05741627 419 32.21957041 420 32.38095238 421 32.30403800 422 32.22748815 423 32.15130024 424 32.31132075 425 32.23529412 426 32.15962441 427 32.08430913 428 32.00934579 429 32.16783217 430 32.09302326 431 32.25058005 432 32.40740741 433 32.33256351 434 32.25806452 435 32.18390805 436 32.11009174 437 32.03661327 438 32.19178082 439 32.34624146 440 32.27272727 441 32.19954649 442 32.12669683 443 32.05417607 444 32.20720721 445 32.13483146 446 32.06278027 447 31.99105145 448 31.91964286 449 31.84855234 450 31.77777778 451 31.70731707 452 31.85840708 453 32.00883002 454 31.93832599 455 31.86813187 456 32.01754386 457 32.16630197 458 32.09606987 459 32.24400871 460 32.39130435 461 32.32104121 462 32.25108225 463 32.39740821 464 32.32758621 465 32.25806452 466 32.18884120 467 32.33404711 468 32.26495726 469 32.40938166 470 32.34042553 471 32.27176221 472 32.20338983 473 32.34672304 474 32.27848101 475 32.21052632 476 32.35294118 477 32.28511530 478 32.42677824 479 32.56784969 480 32.50000000 481 32.43243243 482 32.36514523 483 32.29813665 484 32.23140496 485 32.16494845 486 32.09876543 487 32.23819302 488 32.17213115 489 32.10633947 490 32.24489796 491 32.38289206 492 32.31707317 493 32.25152130 494 32.38866397 495 32.52525253 496 32.66129032 497 32.59557344 498 32.53012048 499 32.46492986 500 32.60000000 501 32.53493014 502 32.66932271 503 32.60437376 504 32.73809524 505 32.67326733 506 32.80632411 507 32.74161736 508 32.67716535 509 32.61296660 510 32.74509804 511 32.68101761 512 32.61718750 513 32.55360624 514 32.49027237 515 32.42718447 516 32.36434109 517 32.49516441 518 32.62548263 519 32.56262042 520 32.50000000 521 32.43761996 522 32.37547893 523 32.50478011 524 32.63358779 525 32.57142857 526 32.50950570 527 32.63757116 528 32.76515152 529 32.89224953 530 33.01886792 531 32.95668550 532 32.89473684 533 33.02063790 534 32.95880150 535 33.08411215 536 33.02238806 537 33.14711359 538 33.27137546 539 33.20964750 540 33.14814815 541 33.08687616 542 33.21033210 543 33.14917127 544 33.08823529 545 33.02752294 546 32.96703297 547 32.90676417 548 33.02919708 549 32.96903461 550 32.90909091 551 33.03085299 552 32.97101449 553 32.91139241 554 32.85198556 555 32.79279279 556 32.73381295 557 32.67504488 558 32.79569892 559 32.73703041 560 32.67857143 561 32.62032086 562 32.56227758 563 32.68206039 564 32.62411348 565 32.74336283 566 32.68551237 567 32.62786596 568 32.74647887 569 32.68892794 570 32.63157895 571 32.74956217 572 32.69230769 573 32.63525305 574 32.75261324 575 32.69565217 576 32.81250000 577 32.92894281 578 32.87197232 579 32.81519862 580 32.75862069 581 32.70223752 582 32.64604811 583 32.76157804 584 32.87671233 585 32.82051282 586 32.76450512 587 32.70868825 588 32.65306122 589 32.59762309 590 32.54237288 591 32.48730964 592 32.43243243 593 32.37774030 594 32.49158249 595 32.43697479 596 32.55033557 597 32.49581240 598 32.44147157 599 32.55425710 600 32.50000000 601 32.61231281 602 32.55813953 603 32.66998342 604 32.61589404 605 32.72727273 606 32.67326733 607 32.78418451 608 32.73026316 609 32.67651888 610 32.62295082 611 32.56955810 612 32.51633987 613 32.46329527 614 32.57328990 615 32.52032520 616 32.62987013 617 32.57698541 618 32.52427184 619 32.47172859 620 32.41935484 621 32.36714976 622 32.31511254 623 32.26324238 624 32.21153846 625 32.16000000 626 32.26837061 627 32.21690590 628 32.16560510 629 32.27344992 630 32.38095238 631 32.32963550 632 32.43670886 633 32.38546603 634 32.33438486 635 32.28346457 636 32.23270440 637 32.33908948 638 32.28840125 639 32.23787167 640 32.34375000 641 32.29329173 642 32.39875389 643 32.34836703 644 32.45341615 645 32.40310078 646 32.35294118 647 32.30293663 648 32.25308642 649 32.20338983 650 32.15384615 651 32.10445469 652 32.20858896 653 32.15926493 654 32.26299694 655 32.21374046 656 32.31707317 657 32.26788432 658 32.37082067 659 32.32169954 660 32.27272727 661 32.22390318 662 32.17522659 663 32.12669683 664 32.07831325 665 32.03007519 666 31.98198198 667 31.93403298 668 32.03592814 669 31.98804185 670 32.08955224 671 32.04172876 672 31.99404762 673 31.94650817 674 31.89910979 675 31.85185185 676 31.80473373 677 31.75775480 678 31.71091445 679 31.66421208 680 31.76470588 681 31.86490455 682 31.81818182 683 31.91800878 684 31.87134503 685 31.82481752 686 31.77842566 687 31.87772926 688 31.83139535 689 31.78519594 690 31.73913043 691 31.83791606 692 31.79190751 693 31.74603175 694 31.84438040 695 31.79856115 696 31.89655172 697 31.85078910 698 31.80515759 699 31.90271817 700 31.85714286 701 31.95435093 702 32.05128205 703 32.14793741 704 32.10227273 705 32.05673759 706 32.01133144 707 31.96605375 708 31.92090395 709 31.87588152 710 31.97183099 711 32.06751055 712 32.02247191 713 32.11781206 714 32.07282913 715 32.02797203 716 32.12290503 717 32.07810321 718 32.03342618 719 31.98887344 720 31.94444444 721 31.90013870 722 31.85595568 723 31.95020747 724 31.90607735 725 32.00000000 726 31.95592287 727 31.91196699 728 31.86813187 729 31.82441701 730 31.78082192 731 31.73734610 732 31.83060109 733 31.92360164 734 32.01634877 735 31.97278912 736 31.92934783 737 32.02170963 738 32.11382114 739 32.20568336 740 32.29729730 741 32.25371120 742 32.34501348 743 32.43606999 744 32.52688172 745 32.48322148 746 32.43967828 747 32.39625167 748 32.35294118 749 32.30974633 750 32.40000000 Final result: 32.4000 ±1.7100 Random chance: 25.0000 ±1.5822