common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 869 tasks in prompt multiple_choice_score: selecting 750 random tasks from 869 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 100.00000000 3 100.00000000 4 75.00000000 5 80.00000000 6 83.33333333 7 85.71428571 8 87.50000000 9 77.77777778 10 80.00000000 11 72.72727273 12 75.00000000 13 69.23076923 14 71.42857143 15 73.33333333 16 75.00000000 17 76.47058824 18 72.22222222 19 73.68421053 20 70.00000000 21 71.42857143 22 72.72727273 23 69.56521739 24 70.83333333 25 68.00000000 26 69.23076923 27 70.37037037 28 71.42857143 29 68.96551724 30 70.00000000 31 67.74193548 32 65.62500000 33 63.63636364 34 61.76470588 35 60.00000000 36 58.33333333 37 59.45945946 38 57.89473684 39 58.97435897 40 60.00000000 41 60.97560976 42 61.90476190 43 60.46511628 44 59.09090909 45 60.00000000 46 60.86956522 47 61.70212766 48 60.41666667 49 59.18367347 50 58.00000000 51 58.82352941 52 59.61538462 53 58.49056604 54 57.40740741 55 56.36363636 56 57.14285714 57 57.89473684 58 58.62068966 59 59.32203390 60 60.00000000 61 60.65573770 62 59.67741935 63 60.31746032 64 60.93750000 65 60.00000000 66 60.60606061 67 61.19402985 68 61.76470588 69 62.31884058 70 61.42857143 71 60.56338028 72 59.72222222 73 60.27397260 74 59.45945946 75 60.00000000 76 60.52631579 77 59.74025974 78 60.25641026 79 59.49367089 80 60.00000000 81 59.25925926 82 59.75609756 83 59.03614458 84 59.52380952 85 60.00000000 86 60.46511628 87 59.77011494 88 60.22727273 89 60.67415730 90 61.11111111 91 61.53846154 92 61.95652174 93 61.29032258 94 60.63829787 95 61.05263158 96 61.45833333 97 60.82474227 98 61.22448980 99 61.61616162 100 61.00000000 101 60.39603960 102 59.80392157 103 59.22330097 104 59.61538462 105 60.00000000 106 59.43396226 107 59.81308411 108 59.25925926 109 58.71559633 110 59.09090909 111 59.45945946 112 59.82142857 113 60.17699115 114 60.52631579 115 60.86956522 116 60.34482759 117 59.82905983 118 60.16949153 119 59.66386555 120 60.00000000 121 59.50413223 122 59.83606557 123 60.16260163 124 59.67741935 125 60.00000000 126 60.31746032 127 60.62992126 128 60.93750000 129 60.46511628 130 60.00000000 131 60.30534351 132 59.84848485 133 60.15037594 134 59.70149254 135 59.25925926 136 58.82352941 137 59.12408759 138 59.42028986 139 59.71223022 140 60.00000000 141 60.28368794 142 60.56338028 143 60.83916084 144 61.11111111 145 61.37931034 146 60.95890411 147 60.54421769 148 60.81081081 149 61.07382550 150 61.33333333 151 60.92715232 152 61.18421053 153 61.43790850 154 61.03896104 155 60.64516129 156 60.89743590 157 60.50955414 158 60.12658228 159 60.37735849 160 60.62500000 161 60.86956522 162 61.11111111 163 61.34969325 164 60.97560976 165 60.60606061 166 60.84337349 167 60.47904192 168 60.71428571 169 60.35502959 170 60.58823529 171 60.23391813 172 60.46511628 173 60.69364162 174 60.91954023 175 60.57142857 176 60.79545455 177 61.01694915 178 60.67415730 179 60.89385475 180 61.11111111 181 60.77348066 182 60.98901099 183 60.65573770 184 60.86956522 185 61.08108108 186 61.29032258 187 61.49732620 188 61.17021277 189 60.84656085 190 60.52631579 191 60.73298429 192 60.93750000 193 60.62176166 194 60.82474227 195 61.02564103 196 61.22448980 197 61.42131980 198 61.61616162 199 61.30653266 200 61.50000000 201 61.69154229 202 61.88118812 203 62.06896552 204 62.25490196 205 61.95121951 206 61.65048544 207 61.35265700 208 61.53846154 209 61.72248804 210 61.90476190 211 62.08530806 212 62.26415094 213 62.44131455 214 62.61682243 215 62.79069767 216 62.96296296 217 62.67281106 218 62.84403670 219 62.55707763 220 62.27272727 221 61.99095023 222 62.16216216 223 61.88340807 224 61.60714286 225 61.33333333 226 61.50442478 227 61.67400881 228 61.84210526 229 62.00873362 230 62.17391304 231 61.90476190 232 61.63793103 233 61.80257511 234 61.53846154 235 61.27659574 236 61.01694915 237 61.18143460 238 61.34453782 239 61.50627615 240 61.66666667 241 61.82572614 242 61.57024793 243 61.72839506 244 61.47540984 245 61.63265306 246 61.78861789 247 61.94331984 248 62.09677419 249 61.84738956 250 62.00000000 251 62.15139442 252 62.30158730 253 62.45059289 254 62.20472441 255 62.35294118 256 62.10937500 257 61.86770428 258 62.01550388 259 61.77606178 260 61.92307692 261 62.06896552 262 62.21374046 263 62.35741445 264 62.50000000 265 62.26415094 266 62.03007519 267 62.17228464 268 62.31343284 269 62.45353160 270 62.59259259 271 62.36162362 272 62.13235294 273 61.90476190 274 61.67883212 275 61.45454545 276 61.59420290 277 61.73285199 278 61.51079137 279 61.64874552 280 61.78571429 281 61.92170819 282 62.05673759 283 61.83745583 284 61.97183099 285 62.10526316 286 62.23776224 287 62.36933798 288 62.50000000 289 62.28373702 290 62.41379310 291 62.54295533 292 62.67123288 293 62.79863481 294 62.58503401 295 62.71186441 296 62.83783784 297 62.96296296 298 63.08724832 299 63.21070234 300 63.33333333 301 63.12292359 302 63.24503311 303 63.36633663 304 63.48684211 305 63.27868852 306 63.07189542 307 63.19218241 308 63.31168831 309 63.43042071 310 63.54838710 311 63.66559486 312 63.46153846 313 63.57827476 314 63.37579618 315 63.17460317 316 63.29113924 317 63.40694006 318 63.20754717 319 63.00940439 320 63.12500000 321 63.23987539 322 63.04347826 323 63.15789474 324 62.96296296 325 63.07692308 326 63.19018405 327 62.99694190 328 62.80487805 329 62.91793313 330 62.72727273 331 62.83987915 332 62.65060241 333 62.76276276 334 62.57485030 335 62.38805970 336 62.20238095 337 62.31454006 338 62.13017751 339 61.94690265 340 61.76470588 341 61.58357771 342 61.40350877 343 61.51603499 344 61.62790698 345 61.44927536 346 61.27167630 347 61.09510086 348 60.91954023 349 61.03151862 350 60.85714286 351 60.68376068 352 60.51136364 353 60.62322946 354 60.45197740 355 60.28169014 356 60.39325843 357 60.50420168 358 60.61452514 359 60.44568245 360 60.55555556 361 60.66481994 362 60.77348066 363 60.60606061 364 60.43956044 365 60.54794521 366 60.65573770 367 60.76294278 368 60.86956522 369 60.97560976 370 61.08108108 371 60.91644205 372 61.02150538 373 60.85790885 374 60.96256684 375 60.80000000 376 60.90425532 377 61.00795756 378 61.11111111 379 60.94986807 380 61.05263158 381 61.15485564 382 60.99476440 383 60.83550914 384 60.93750000 385 60.77922078 386 60.88082902 387 60.72351421 388 60.56701031 389 60.66838046 390 60.51282051 391 60.35805627 392 60.45918367 393 60.55979644 394 60.65989848 395 60.75949367 396 60.60606061 397 60.70528967 398 60.55276382 399 60.65162907 400 60.75000000 401 60.84788030 402 60.94527363 403 61.04218362 404 61.13861386 405 61.23456790 406 61.33004926 407 61.42506143 408 61.51960784 409 61.36919315 410 61.46341463 411 61.55717762 412 61.40776699 413 61.25907990 414 61.35265700 415 61.20481928 416 61.29807692 417 61.15107914 418 61.00478469 419 60.85918854 420 60.95238095 421 60.80760095 422 60.66350711 423 60.75650118 424 60.61320755 425 60.70588235 426 60.79812207 427 60.88992974 428 60.74766355 429 60.83916084 430 60.93023256 431 61.02088167 432 61.11111111 433 60.96997691 434 61.05990783 435 61.14942529 436 61.23853211 437 61.32723112 438 61.18721461 439 61.27562642 440 61.13636364 441 60.99773243 442 60.85972851 443 60.72234763 444 60.58558559 445 60.67415730 446 60.76233184 447 60.85011186 448 60.93750000 449 61.02449889 450 60.88888889 451 60.75388027 452 60.84070796 453 60.92715232 454 61.01321586 455 61.09890110 456 61.18421053 457 61.26914661 458 61.35371179 459 61.43790850 460 61.52173913 461 61.38828633 462 61.47186147 463 61.33909287 464 61.20689655 465 61.29032258 466 61.37339056 467 61.45610278 468 61.53846154 469 61.40724947 470 61.48936170 471 61.57112527 472 61.65254237 473 61.52219873 474 61.60337553 475 61.68421053 476 61.76470588 477 61.84486373 478 61.92468619 479 62.00417537 480 62.08333333 481 62.16216216 482 62.24066390 483 62.31884058 484 62.39669421 485 62.47422680 486 62.55144033 487 62.42299795 488 62.50000000 489 62.37218814 490 62.24489796 491 62.11812627 492 62.19512195 493 62.06896552 494 62.14574899 495 62.02020202 496 61.89516129 497 61.97183099 498 62.04819277 499 62.12424850 500 62.20000000 501 62.07584830 502 62.15139442 503 62.22664016 504 62.10317460 505 62.17821782 506 62.05533597 507 61.93293886 508 62.00787402 509 62.08251473 510 62.15686275 511 62.03522505 512 61.91406250 513 61.79337232 514 61.86770428 515 61.94174757 516 61.82170543 517 61.89555126 518 61.96911197 519 62.04238921 520 61.92307692 521 61.99616123 522 62.06896552 523 61.95028681 524 61.83206107 525 61.71428571 526 61.59695817 527 61.48007590 528 61.36363636 529 61.24763705 530 61.32075472 531 61.39359699 532 61.46616541 533 61.53846154 534 61.42322097 535 61.30841121 536 61.19402985 537 61.26629423 538 61.15241636 539 61.22448980 540 61.11111111 541 61.18299445 542 61.25461255 543 61.32596685 544 61.39705882 545 61.46788991 546 61.35531136 547 61.24314442 548 61.31386861 549 61.20218579 550 61.27272727 551 61.16152450 552 61.05072464 553 61.12115732 554 61.19133574 555 61.26126126 556 61.15107914 557 61.04129264 558 60.93189964 559 61.00178891 560 61.07142857 561 61.14081996 562 61.20996441 563 61.27886323 564 61.17021277 565 61.06194690 566 61.13074205 567 61.19929453 568 61.26760563 569 61.15992970 570 61.05263158 571 61.12084063 572 61.01398601 573 61.08202443 574 61.14982578 575 61.04347826 576 61.11111111 577 61.17850953 578 61.24567474 579 61.31260794 580 61.20689655 581 61.27366609 582 61.34020619 583 61.23499142 584 61.30136986 585 61.36752137 586 61.26279863 587 61.32879046 588 61.39455782 589 61.46010187 590 61.52542373 591 61.59052453 592 61.65540541 593 61.72006745 594 61.78451178 595 61.84873950 596 61.74496644 597 61.64154104 598 61.70568562 599 61.76961603 600 61.66666667 601 61.73044925 602 61.62790698 603 61.52570481 604 61.58940397 605 61.65289256 606 61.71617162 607 61.77924217 608 61.84210526 609 61.74055829 610 61.80327869 611 61.70212766 612 61.60130719 613 61.50081566 614 61.56351792 615 61.62601626 616 61.52597403 617 61.58833063 618 61.48867314 619 61.38933764 620 61.45161290 621 61.51368760 622 61.41479100 623 61.47672552 624 61.37820513 625 61.28000000 626 61.18210863 627 61.24401914 628 61.30573248 629 61.20826709 630 61.26984127 631 61.33122029 632 61.23417722 633 61.13744076 634 61.19873817 635 61.25984252 636 61.32075472 637 61.22448980 638 61.28526646 639 61.34585290 640 61.25000000 641 61.31045242 642 61.21495327 643 61.27527216 644 61.33540373 645 61.24031008 646 61.30030960 647 61.36012365 648 61.26543210 649 61.17103236 650 61.23076923 651 61.29032258 652 61.34969325 653 61.25574273 654 61.16207951 655 61.06870229 656 60.97560976 657 60.88280061 658 60.79027356 659 60.69802731 660 60.75757576 661 60.81694402 662 60.87613293 663 60.93514329 664 60.99397590 665 61.05263158 666 61.11111111 667 61.16941529 668 61.07784431 669 60.98654709 670 61.04477612 671 60.95380030 672 61.01190476 673 60.92124814 674 60.97922849 675 60.88888889 676 60.79881657 677 60.85672083 678 60.91445428 679 60.82474227 680 60.88235294 681 60.79295154 682 60.70381232 683 60.61493411 684 60.52631579 685 60.58394161 686 60.49562682 687 60.40756914 688 60.46511628 689 60.37735849 690 60.43478261 691 60.49204052 692 60.54913295 693 60.46176046 694 60.37463977 695 60.28776978 696 60.20114943 697 60.11477762 698 60.02865330 699 59.94277539 700 60.00000000 701 60.05706134 702 60.11396011 703 60.17069701 704 60.22727273 705 60.28368794 706 60.33994334 707 60.39603960 708 60.45197740 709 60.50775740 710 60.56338028 711 60.47819972 712 60.53370787 713 60.44880785 714 60.36414566 715 60.27972028 716 60.33519553 717 60.39051604 718 60.30640669 719 60.22253129 720 60.13888889 721 60.19417476 722 60.24930748 723 60.30428769 724 60.22099448 725 60.27586207 726 60.19283747 727 60.11004127 728 60.16483516 729 60.21947874 730 60.27397260 731 60.32831737 732 60.38251366 733 60.43656207 734 60.49046322 735 60.40816327 736 60.46195652 737 60.37991859 738 60.43360434 739 60.48714479 740 60.40540541 741 60.45883941 742 60.37735849 743 60.43068641 744 60.34946237 745 60.40268456 746 60.45576408 747 60.37483266 748 60.29411765 749 60.21361816 750 60.26666667 Final result: 60.2667 +/- 1.7880 Random chance: 25.0083 +/- 1.5824