common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 817 tasks in prompt multiple_choice_score: selecting 750 random tasks from 817 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 50.00000000 3 33.33333333 4 25.00000000 5 20.00000000 6 16.66666667 7 14.28571429 8 12.50000000 9 11.11111111 10 10.00000000 11 9.09090909 12 16.66666667 13 15.38461538 14 14.28571429 15 13.33333333 16 18.75000000 17 17.64705882 18 16.66666667 19 21.05263158 20 20.00000000 21 19.04761905 22 18.18181818 23 17.39130435 24 20.83333333 25 20.00000000 26 23.07692308 27 22.22222222 28 21.42857143 29 24.13793103 30 23.33333333 31 25.80645161 32 25.00000000 33 27.27272727 34 26.47058824 35 25.71428571 36 25.00000000 37 27.02702703 38 26.31578947 39 25.64102564 40 25.00000000 41 24.39024390 42 23.80952381 43 25.58139535 44 27.27272727 45 26.66666667 46 28.26086957 47 27.65957447 48 29.16666667 49 28.57142857 50 28.00000000 51 27.45098039 52 26.92307692 53 28.30188679 54 29.62962963 55 30.90909091 56 30.35714286 57 29.82456140 58 29.31034483 59 28.81355932 60 30.00000000 61 29.50819672 62 30.64516129 63 30.15873016 64 29.68750000 65 29.23076923 66 28.78787879 67 28.35820896 68 27.94117647 69 27.53623188 70 28.57142857 71 29.57746479 72 30.55555556 73 31.50684932 74 31.08108108 75 30.66666667 76 31.57894737 77 31.16883117 78 30.76923077 79 30.37974684 80 30.00000000 81 29.62962963 82 30.48780488 83 30.12048193 84 29.76190476 85 29.41176471 86 29.06976744 87 28.73563218 88 28.40909091 89 28.08988764 90 27.77777778 91 28.57142857 92 28.26086957 93 27.95698925 94 27.65957447 95 28.42105263 96 29.16666667 97 28.86597938 98 28.57142857 99 29.29292929 100 30.00000000 101 30.69306931 102 30.39215686 103 31.06796117 104 30.76923077 105 30.47619048 106 31.13207547 107 31.77570093 108 32.40740741 109 32.11009174 110 32.72727273 111 32.43243243 112 32.14285714 113 31.85840708 114 32.45614035 115 32.17391304 116 31.89655172 117 31.62393162 118 31.35593220 119 31.09243697 120 31.66666667 121 32.23140496 122 31.96721311 123 32.52032520 124 33.06451613 125 33.60000000 126 34.12698413 127 33.85826772 128 33.59375000 129 33.33333333 130 33.07692308 131 32.82442748 132 32.57575758 133 32.33082707 134 32.83582090 135 33.33333333 136 33.08823529 137 33.57664234 138 34.05797101 139 33.81294964 140 34.28571429 141 34.04255319 142 33.80281690 143 33.56643357 144 34.02777778 145 33.79310345 146 33.56164384 147 34.01360544 148 33.78378378 149 33.55704698 150 33.33333333 151 33.77483444 152 33.55263158 153 33.33333333 154 33.11688312 155 32.90322581 156 33.33333333 157 33.12101911 158 33.54430380 159 33.33333333 160 33.12500000 161 32.91925466 162 32.71604938 163 33.12883436 164 32.92682927 165 33.33333333 166 33.73493976 167 33.53293413 168 33.33333333 169 33.13609467 170 32.94117647 171 32.74853801 172 32.55813953 173 32.36994220 174 32.18390805 175 32.00000000 176 32.38636364 177 32.20338983 178 32.02247191 179 31.84357542 180 31.66666667 181 31.49171271 182 31.31868132 183 31.14754098 184 30.97826087 185 30.81081081 186 31.18279570 187 31.01604278 188 30.85106383 189 31.21693122 190 31.05263158 191 30.89005236 192 30.72916667 193 30.56994819 194 30.92783505 195 30.76923077 196 30.61224490 197 30.96446701 198 31.31313131 199 31.15577889 200 31.50000000 201 31.34328358 202 31.68316832 203 31.52709360 204 31.37254902 205 31.21951220 206 31.06796117 207 31.40096618 208 31.25000000 209 31.10047847 210 30.95238095 211 30.80568720 212 30.66037736 213 30.51643192 214 30.37383178 215 30.23255814 216 30.09259259 217 29.95391705 218 30.27522936 219 30.13698630 220 30.00000000 221 29.86425339 222 29.72972973 223 29.59641256 224 29.91071429 225 29.77777778 226 29.64601770 227 29.51541850 228 29.38596491 229 29.25764192 230 29.13043478 231 29.00432900 232 29.31034483 233 29.61373391 234 29.91452991 235 30.21276596 236 30.50847458 237 30.80168776 238 31.09243697 239 30.96234310 240 30.83333333 241 30.70539419 242 30.57851240 243 30.45267490 244 30.32786885 245 30.20408163 246 30.08130081 247 29.95951417 248 30.24193548 249 30.12048193 250 30.00000000 251 30.27888446 252 30.15873016 253 30.03952569 254 29.92125984 255 29.80392157 256 29.68750000 257 29.57198444 258 29.45736434 259 29.34362934 260 29.23076923 261 29.11877395 262 29.00763359 263 28.89733840 264 29.16666667 265 29.43396226 266 29.32330827 267 29.58801498 268 29.47761194 269 29.36802974 270 29.62962963 271 29.52029520 272 29.41176471 273 29.30402930 274 29.19708029 275 29.09090909 276 28.98550725 277 28.88086643 278 28.77697842 279 28.67383513 280 28.57142857 281 28.82562278 282 28.72340426 283 28.62190813 284 28.52112676 285 28.42105263 286 28.67132867 287 28.57142857 288 28.47222222 289 28.37370242 290 28.27586207 291 28.52233677 292 28.76712329 293 29.01023891 294 28.91156463 295 29.15254237 296 29.39189189 297 29.62962963 298 29.53020134 299 29.43143813 300 29.33333333 301 29.23588040 302 29.47019868 303 29.37293729 304 29.27631579 305 29.18032787 306 29.08496732 307 28.99022801 308 29.22077922 309 29.12621359 310 29.03225806 311 28.93890675 312 29.16666667 313 29.07348243 314 28.98089172 315 28.88888889 316 28.79746835 317 28.70662461 318 28.61635220 319 28.52664577 320 28.43750000 321 28.34890966 322 28.57142857 323 28.48297214 324 28.39506173 325 28.30769231 326 28.22085890 327 28.13455657 328 28.35365854 329 28.26747720 330 28.48484848 331 28.39879154 332 28.61445783 333 28.82882883 334 28.74251497 335 28.95522388 336 29.16666667 337 29.08011869 338 28.99408284 339 29.20353982 340 29.11764706 341 29.32551320 342 29.53216374 343 29.44606414 344 29.65116279 345 29.85507246 346 30.05780347 347 30.25936599 348 30.17241379 349 30.08595989 350 30.00000000 351 29.91452991 352 30.11363636 353 30.31161473 354 30.22598870 355 30.42253521 356 30.33707865 357 30.25210084 358 30.44692737 359 30.36211699 360 30.27777778 361 30.19390582 362 30.11049724 363 30.02754821 364 29.94505495 365 30.13698630 366 30.05464481 367 30.24523161 368 30.16304348 369 30.08130081 370 30.00000000 371 29.91913747 372 30.10752688 373 30.02680965 374 30.21390374 375 30.13333333 376 30.05319149 377 29.97347480 378 29.89417989 379 29.81530343 380 29.73684211 381 29.65879265 382 29.58115183 383 29.50391645 384 29.42708333 385 29.35064935 386 29.27461140 387 29.19896641 388 29.38144330 389 29.30591260 390 29.23076923 391 29.41176471 392 29.33673469 393 29.51653944 394 29.69543147 395 29.62025316 396 29.54545455 397 29.72292191 398 29.64824121 399 29.57393484 400 29.50000000 401 29.42643392 402 29.60199005 403 29.52853598 404 29.45544554 405 29.38271605 406 29.31034483 407 29.23832924 408 29.16666667 409 29.09535452 410 29.02439024 411 28.95377129 412 29.12621359 413 29.05569007 414 29.22705314 415 29.15662651 416 29.08653846 417 29.01678657 418 28.94736842 419 28.87828162 420 29.04761905 421 29.21615202 422 29.14691943 423 29.07801418 424 29.00943396 425 28.94117647 426 28.87323944 427 28.80562061 428 28.73831776 429 28.67132867 430 28.60465116 431 28.53828306 432 28.70370370 433 28.86836028 434 29.03225806 435 28.96551724 436 28.89908257 437 28.83295195 438 28.76712329 439 28.70159453 440 28.86363636 441 28.79818594 442 28.73303167 443 28.66817156 444 28.82882883 445 28.76404494 446 28.69955157 447 28.63534676 448 28.57142857 449 28.73051225 450 28.66666667 451 28.60310421 452 28.53982301 453 28.47682119 454 28.41409692 455 28.35164835 456 28.28947368 457 28.22757112 458 28.16593886 459 28.10457516 460 28.04347826 461 27.98264642 462 27.92207792 463 27.86177106 464 27.80172414 465 27.95698925 466 27.89699571 467 28.05139186 468 27.99145299 469 27.93176972 470 27.87234043 471 27.81316348 472 27.75423729 473 27.90697674 474 28.05907173 475 28.21052632 476 28.15126050 477 28.30188679 478 28.24267782 479 28.18371608 480 28.12500000 481 28.06652807 482 28.00829876 483 27.95031056 484 28.09917355 485 28.24742268 486 28.18930041 487 28.33675565 488 28.27868852 489 28.42535787 490 28.57142857 491 28.51323829 492 28.65853659 493 28.60040568 494 28.54251012 495 28.68686869 496 28.62903226 497 28.57142857 498 28.51405622 499 28.45691383 500 28.40000000 501 28.34331337 502 28.28685259 503 28.23061630 504 28.37301587 505 28.51485149 506 28.45849802 507 28.40236686 508 28.34645669 509 28.48722986 510 28.62745098 511 28.57142857 512 28.51562500 513 28.46003899 514 28.40466926 515 28.54368932 516 28.48837209 517 28.62669246 518 28.57142857 519 28.51637765 520 28.46153846 521 28.59884837 522 28.54406130 523 28.68068834 524 28.62595420 525 28.57142857 526 28.51711027 527 28.65275142 528 28.59848485 529 28.54442344 530 28.49056604 531 28.62523540 532 28.57142857 533 28.70544090 534 28.65168539 535 28.59813084 536 28.54477612 537 28.67783985 538 28.62453532 539 28.57142857 540 28.70370370 541 28.65064695 542 28.59778598 543 28.54511971 544 28.49264706 545 28.44036697 546 28.57142857 547 28.51919561 548 28.64963504 549 28.59744991 550 28.54545455 551 28.49364791 552 28.62318841 553 28.75226040 554 28.70036101 555 28.64864865 556 28.77697842 557 28.72531418 558 28.67383513 559 28.62254025 560 28.57142857 561 28.52049911 562 28.46975089 563 28.41918295 564 28.54609929 565 28.67256637 566 28.62190813 567 28.57142857 568 28.52112676 569 28.47100176 570 28.42105263 571 28.37127846 572 28.32167832 573 28.27225131 574 28.39721254 575 28.52173913 576 28.47222222 577 28.42287695 578 28.54671280 579 28.49740933 580 28.62068966 581 28.74354561 582 28.86597938 583 28.81646655 584 28.93835616 585 28.88888889 586 29.01023891 587 29.13117547 588 29.08163265 589 29.03225806 590 29.15254237 591 29.10321489 592 29.05405405 593 29.00505902 594 29.12457912 595 29.07563025 596 29.02684564 597 29.14572864 598 29.09698997 599 29.04841402 600 29.00000000 601 29.11813644 602 29.06976744 603 29.02155887 604 28.97350993 605 28.92561983 606 29.04290429 607 28.99505766 608 28.94736842 609 28.89983580 610 28.85245902 611 28.80523732 612 28.92156863 613 28.87438825 614 28.82736156 615 28.78048780 616 28.89610390 617 29.01134522 618 28.96440129 619 29.07915994 620 29.03225806 621 28.98550725 622 29.09967846 623 29.05296950 624 29.00641026 625 28.96000000 626 28.91373802 627 29.02711324 628 28.98089172 629 28.93481717 630 28.88888889 631 29.00158479 632 28.95569620 633 28.90995261 634 29.02208202 635 28.97637795 636 28.93081761 637 29.04238619 638 28.99686520 639 28.95148670 640 28.90625000 641 28.86115445 642 28.97196262 643 28.92690513 644 29.03726708 645 29.14728682 646 29.10216718 647 29.05718702 648 29.16666667 649 29.12172573 650 29.23076923 651 29.33947773 652 29.44785276 653 29.55589587 654 29.51070336 655 29.46564885 656 29.42073171 657 29.52815830 658 29.48328267 659 29.43854325 660 29.39393939 661 29.34947050 662 29.30513595 663 29.26093514 664 29.21686747 665 29.32330827 666 29.42942943 667 29.38530735 668 29.34131737 669 29.29745889 670 29.25373134 671 29.21013413 672 29.16666667 673 29.12332838 674 29.08011869 675 29.18518519 676 29.28994083 677 29.24667651 678 29.20353982 679 29.16053019 680 29.26470588 681 29.22173275 682 29.17888563 683 29.13616398 684 29.23976608 685 29.19708029 686 29.15451895 687 29.11208151 688 29.21511628 689 29.17271408 690 29.13043478 691 29.08827786 692 29.04624277 693 29.14862915 694 29.25072046 695 29.20863309 696 29.31034483 697 29.26829268 698 29.22636103 699 29.32761087 700 29.42857143 701 29.38659058 702 29.34472934 703 29.30298720 704 29.40340909 705 29.36170213 706 29.32011331 707 29.27864215 708 29.23728814 709 29.19605078 710 29.15492958 711 29.11392405 712 29.21348315 713 29.17251052 714 29.13165266 715 29.09090909 716 29.05027933 717 29.00976290 718 28.96935933 719 29.06815021 720 29.02777778 721 28.98751734 722 28.94736842 723 28.90733057 724 28.86740331 725 28.82758621 726 28.92561983 727 28.88583219 728 28.84615385 729 28.80658436 730 28.90410959 731 28.86456908 732 28.96174863 733 28.92223738 734 28.88283379 735 28.84353741 736 28.80434783 737 28.76526459 738 28.72628726 739 28.68741543 740 28.64864865 741 28.60998650 742 28.57142857 743 28.53297443 744 28.62903226 745 28.59060403 746 28.68632708 747 28.64792503 748 28.60962567 749 28.70493992 750 28.66666667 Final result: 28.6667 ±1.6523 Random chance: 19.8992 ±1.4588