common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 817 tasks in prompt multiple_choice_score: selecting 750 random tasks from 817 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 25.00000000 5 20.00000000 6 16.66666667 7 14.28571429 8 12.50000000 9 11.11111111 10 10.00000000 11 9.09090909 12 16.66666667 13 15.38461538 14 14.28571429 15 13.33333333 16 18.75000000 17 17.64705882 18 16.66666667 19 21.05263158 20 20.00000000 21 19.04761905 22 18.18181818 23 17.39130435 24 20.83333333 25 20.00000000 26 23.07692308 27 22.22222222 28 21.42857143 29 24.13793103 30 23.33333333 31 25.80645161 32 25.00000000 33 27.27272727 34 26.47058824 35 25.71428571 36 25.00000000 37 27.02702703 38 26.31578947 39 25.64102564 40 25.00000000 41 24.39024390 42 23.80952381 43 25.58139535 44 25.00000000 45 24.44444444 46 26.08695652 47 25.53191489 48 27.08333333 49 26.53061224 50 28.00000000 51 27.45098039 52 26.92307692 53 28.30188679 54 29.62962963 55 30.90909091 56 30.35714286 57 29.82456140 58 29.31034483 59 28.81355932 60 30.00000000 61 29.50819672 62 30.64516129 63 30.15873016 64 29.68750000 65 29.23076923 66 28.78787879 67 28.35820896 68 27.94117647 69 27.53623188 70 28.57142857 71 29.57746479 72 30.55555556 73 31.50684932 74 31.08108108 75 30.66666667 76 31.57894737 77 31.16883117 78 30.76923077 79 30.37974684 80 30.00000000 81 29.62962963 82 29.26829268 83 28.91566265 84 28.57142857 85 28.23529412 86 27.90697674 87 27.58620690 88 27.27272727 89 28.08988764 90 27.77777778 91 27.47252747 92 27.17391304 93 26.88172043 94 26.59574468 95 27.36842105 96 28.12500000 97 27.83505155 98 27.55102041 99 28.28282828 100 29.00000000 101 29.70297030 102 29.41176471 103 30.09708738 104 29.80769231 105 29.52380952 106 30.18867925 107 30.84112150 108 31.48148148 109 31.19266055 110 31.81818182 111 31.53153153 112 31.25000000 113 30.97345133 114 31.57894737 115 31.30434783 116 31.03448276 117 30.76923077 118 30.50847458 119 30.25210084 120 30.00000000 121 30.57851240 122 30.32786885 123 30.89430894 124 31.45161290 125 32.00000000 126 32.53968254 127 32.28346457 128 32.03125000 129 31.78294574 130 31.53846154 131 31.29770992 132 31.06060606 133 30.82706767 134 31.34328358 135 31.85185185 136 31.61764706 137 32.11678832 138 32.60869565 139 33.09352518 140 33.57142857 141 33.33333333 142 33.09859155 143 32.86713287 144 33.33333333 145 33.10344828 146 32.87671233 147 33.33333333 148 33.10810811 149 32.88590604 150 32.66666667 151 33.11258278 152 32.89473684 153 32.67973856 154 32.46753247 155 32.25806452 156 32.69230769 157 32.48407643 158 32.91139241 159 32.70440252 160 32.50000000 161 32.29813665 162 32.09876543 163 32.51533742 164 32.31707317 165 32.72727273 166 33.13253012 167 32.93413174 168 32.73809524 169 32.54437870 170 32.35294118 171 32.16374269 172 31.97674419 173 32.36994220 174 32.18390805 175 32.00000000 176 32.38636364 177 32.20338983 178 32.02247191 179 31.84357542 180 31.66666667 181 31.49171271 182 31.31868132 183 31.14754098 184 30.97826087 185 30.81081081 186 31.18279570 187 31.01604278 188 30.85106383 189 31.21693122 190 31.05263158 191 30.89005236 192 30.72916667 193 30.56994819 194 30.92783505 195 30.76923077 196 30.61224490 197 30.96446701 198 31.31313131 199 31.15577889 200 31.50000000 201 31.34328358 202 31.68316832 203 31.52709360 204 31.37254902 205 31.70731707 206 31.55339806 207 31.88405797 208 31.73076923 209 32.05741627 210 31.90476190 211 31.75355450 212 31.60377358 213 31.45539906 214 31.30841121 215 31.16279070 216 31.01851852 217 30.87557604 218 31.19266055 219 31.05022831 220 30.90909091 221 30.76923077 222 30.63063063 223 30.49327354 224 30.80357143 225 30.66666667 226 30.53097345 227 30.39647577 228 30.26315789 229 30.13100437 230 30.00000000 231 29.87012987 232 29.74137931 233 30.04291845 234 29.91452991 235 29.78723404 236 30.08474576 237 30.37974684 238 30.67226891 239 30.54393305 240 30.41666667 241 30.29045643 242 30.16528926 243 30.04115226 244 29.91803279 245 29.79591837 246 29.67479675 247 29.55465587 248 29.83870968 249 30.12048193 250 30.00000000 251 30.27888446 252 30.15873016 253 30.03952569 254 29.92125984 255 29.80392157 256 29.68750000 257 29.57198444 258 29.45736434 259 29.34362934 260 29.23076923 261 29.50191571 262 29.38931298 263 29.27756654 264 29.54545455 265 29.81132075 266 29.69924812 267 29.96254682 268 29.85074627 269 29.73977695 270 29.62962963 271 29.52029520 272 29.41176471 273 29.30402930 274 29.19708029 275 29.09090909 276 28.98550725 277 28.88086643 278 28.77697842 279 28.67383513 280 28.57142857 281 28.82562278 282 28.72340426 283 28.62190813 284 28.52112676 285 28.42105263 286 28.67132867 287 28.57142857 288 28.47222222 289 28.37370242 290 28.27586207 291 28.52233677 292 28.76712329 293 29.01023891 294 28.91156463 295 29.15254237 296 29.39189189 297 29.62962963 298 29.53020134 299 29.43143813 300 29.33333333 301 29.23588040 302 29.13907285 303 29.04290429 304 28.94736842 305 28.85245902 306 28.75816993 307 28.66449511 308 28.89610390 309 28.80258900 310 28.70967742 311 28.61736334 312 28.84615385 313 28.75399361 314 28.66242038 315 28.57142857 316 28.48101266 317 28.39116719 318 28.30188679 319 28.21316614 320 28.12500000 321 28.03738318 322 28.26086957 323 28.17337461 324 28.08641975 325 28.00000000 326 27.91411043 327 27.82874618 328 28.04878049 329 27.96352584 330 28.18181818 331 28.09667674 332 28.31325301 333 28.52852853 334 28.44311377 335 28.65671642 336 28.86904762 337 28.78338279 338 28.69822485 339 28.90855457 340 28.82352941 341 29.03225806 342 29.23976608 343 29.15451895 344 29.36046512 345 29.56521739 346 29.76878613 347 29.97118156 348 29.88505747 349 30.08595989 350 30.00000000 351 29.91452991 352 30.11363636 353 30.31161473 354 30.22598870 355 30.42253521 356 30.33707865 357 30.25210084 358 30.44692737 359 30.36211699 360 30.27777778 361 30.19390582 362 30.11049724 363 30.30303030 364 30.21978022 365 30.41095890 366 30.32786885 367 30.51771117 368 30.43478261 369 30.35230352 370 30.27027027 371 30.18867925 372 30.37634409 373 30.29490617 374 30.48128342 375 30.40000000 376 30.31914894 377 30.23872679 378 30.42328042 379 30.34300792 380 30.26315789 381 30.18372703 382 30.10471204 383 30.02610966 384 29.94791667 385 29.87012987 386 29.79274611 387 29.71576227 388 29.89690722 389 29.82005141 390 29.74358974 391 29.66751918 392 29.59183673 393 29.51653944 394 29.69543147 395 29.62025316 396 29.54545455 397 29.72292191 398 29.64824121 399 29.57393484 400 29.50000000 401 29.42643392 402 29.60199005 403 29.52853598 404 29.45544554 405 29.38271605 406 29.31034483 407 29.23832924 408 29.16666667 409 29.09535452 410 29.02439024 411 28.95377129 412 29.12621359 413 29.05569007 414 29.22705314 415 29.15662651 416 29.08653846 417 29.01678657 418 28.94736842 419 28.87828162 420 29.04761905 421 29.21615202 422 29.14691943 423 29.07801418 424 29.00943396 425 28.94117647 426 28.87323944 427 28.80562061 428 28.73831776 429 28.67132867 430 28.60465116 431 28.53828306 432 28.70370370 433 28.86836028 434 29.03225806 435 28.96551724 436 29.12844037 437 29.06178490 438 28.99543379 439 28.92938497 440 29.09090909 441 29.02494331 442 29.18552036 443 29.11963883 444 29.27927928 445 29.21348315 446 29.14798206 447 29.08277405 448 29.01785714 449 29.17594655 450 29.11111111 451 29.04656319 452 28.98230088 453 28.91832230 454 28.85462555 455 28.79120879 456 28.72807018 457 28.66520788 458 28.60262009 459 28.54030501 460 28.47826087 461 28.41648590 462 28.35497835 463 28.29373650 464 28.23275862 465 28.38709677 466 28.32618026 467 28.26552463 468 28.20512821 469 28.14498934 470 28.08510638 471 28.02547771 472 27.96610169 473 27.90697674 474 27.84810127 475 28.00000000 476 27.94117647 477 28.09224319 478 28.03347280 479 27.97494781 480 27.91666667 481 27.85862786 482 27.80082988 483 27.95031056 484 28.09917355 485 28.24742268 486 28.18930041 487 28.13141684 488 28.07377049 489 28.22085890 490 28.36734694 491 28.30957230 492 28.45528455 493 28.39756592 494 28.34008097 495 28.48484848 496 28.42741935 497 28.37022133 498 28.31325301 499 28.25651303 500 28.20000000 501 28.34331337 502 28.48605578 503 28.42942346 504 28.57142857 505 28.51485149 506 28.45849802 507 28.40236686 508 28.34645669 509 28.48722986 510 28.62745098 511 28.57142857 512 28.51562500 513 28.46003899 514 28.40466926 515 28.54368932 516 28.48837209 517 28.62669246 518 28.57142857 519 28.51637765 520 28.46153846 521 28.59884837 522 28.54406130 523 28.68068834 524 28.62595420 525 28.76190476 526 28.70722433 527 28.65275142 528 28.78787879 529 28.73345936 530 28.67924528 531 28.81355932 532 28.75939850 533 28.89305816 534 29.02621723 535 28.97196262 536 28.91791045 537 29.05027933 538 28.99628253 539 28.94248609 540 28.88888889 541 28.83548983 542 28.78228782 543 28.72928177 544 28.67647059 545 28.62385321 546 28.75457875 547 28.70201097 548 28.83211679 549 28.77959927 550 28.72727273 551 28.67513612 552 28.80434783 553 28.75226040 554 28.70036101 555 28.64864865 556 28.77697842 557 28.72531418 558 28.67383513 559 28.62254025 560 28.75000000 561 28.69875223 562 28.64768683 563 28.59680284 564 28.72340426 565 28.84955752 566 28.79858657 567 28.74779541 568 28.87323944 569 28.99824253 570 28.94736842 571 28.89667250 572 28.84615385 573 28.79581152 574 28.91986063 575 29.04347826 576 28.99305556 577 28.94280763 578 29.06574394 579 29.01554404 580 29.13793103 581 29.25989673 582 29.20962199 583 29.15951973 584 29.28082192 585 29.23076923 586 29.35153584 587 29.30153322 588 29.25170068 589 29.20203735 590 29.32203390 591 29.27241963 592 29.22297297 593 29.17369309 594 29.12457912 595 29.07563025 596 29.02684564 597 29.14572864 598 29.09698997 599 29.04841402 600 29.00000000 601 29.11813644 602 29.23588040 603 29.18739635 604 29.13907285 605 29.09090909 606 29.20792079 607 29.15980231 608 29.11184211 609 29.06403941 610 29.01639344 611 28.96890344 612 29.08496732 613 29.03752039 614 28.99022801 615 28.94308943 616 29.05844156 617 29.17341977 618 29.12621359 619 29.24071082 620 29.19354839 621 29.14653784 622 29.26045016 623 29.21348315 624 29.16666667 625 29.12000000 626 29.07348243 627 29.18660287 628 29.14012739 629 29.09379968 630 29.04761905 631 29.16006339 632 29.11392405 633 29.06793049 634 29.17981073 635 29.13385827 636 29.24528302 637 29.35635793 638 29.31034483 639 29.26447574 640 29.21875000 641 29.17316693 642 29.28348910 643 29.23794712 644 29.34782609 645 29.45736434 646 29.41176471 647 29.36630603 648 29.47530864 649 29.42989214 650 29.53846154 651 29.64669739 652 29.75460123 653 29.86217458 654 29.81651376 655 29.77099237 656 29.72560976 657 29.83257230 658 29.78723404 659 29.74203338 660 29.69696970 661 29.65204236 662 29.60725076 663 29.56259427 664 29.51807229 665 29.62406015 666 29.72972973 667 29.68515742 668 29.64071856 669 29.59641256 670 29.55223881 671 29.50819672 672 29.46428571 673 29.42050520 674 29.37685460 675 29.48148148 676 29.58579882 677 29.54209749 678 29.49852507 679 29.45508100 680 29.55882353 681 29.51541850 682 29.47214076 683 29.42898975 684 29.53216374 685 29.48905109 686 29.44606414 687 29.40320233 688 29.50581395 689 29.60812772 690 29.56521739 691 29.66714906 692 29.62427746 693 29.72582973 694 29.68299712 695 29.78417266 696 29.88505747 697 29.84218077 698 29.79942693 699 29.89985694 700 30.00000000 701 29.95720399 702 29.91452991 703 29.87197724 704 29.82954545 705 29.78723404 706 29.74504249 707 29.70297030 708 29.66101695 709 29.61918195 710 29.57746479 711 29.67651195 712 29.77528090 713 29.73352034 714 29.69187675 715 29.65034965 716 29.60893855 717 29.56764296 718 29.52646240 719 29.62447844 720 29.58333333 721 29.54230236 722 29.50138504 723 29.46058091 724 29.41988950 725 29.37931034 726 29.47658402 727 29.43603851 728 29.39560440 729 29.35528121 730 29.45205479 731 29.41176471 732 29.50819672 733 29.46793997 734 29.42779292 735 29.38775510 736 29.34782609 737 29.44369064 738 29.40379404 739 29.49932341 740 29.45945946 741 29.55465587 742 29.51482480 743 29.47510094 744 29.56989247 745 29.53020134 746 29.62466488 747 29.58500669 748 29.54545455 749 29.50600801 750 29.46666667 Final result: 29.4667 ±1.6658 Random chance: 19.8992 ±1.4588