common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | multiple_choice_score: there are 869 tasks in prompt multiple_choice_score: selecting 750 random tasks from 869 tasks available multiple_choice_score: preparing task data...done multiple_choice_score : calculating TruthfulQA score over 750 tasks. task acc_norm 1 100.00000000 2 100.00000000 3 66.66666667 4 50.00000000 5 60.00000000 6 66.66666667 7 71.42857143 8 75.00000000 9 66.66666667 10 70.00000000 11 63.63636364 12 66.66666667 13 61.53846154 14 64.28571429 15 66.66666667 16 68.75000000 17 70.58823529 18 66.66666667 19 68.42105263 20 65.00000000 21 66.66666667 22 68.18181818 23 65.21739130 24 66.66666667 25 64.00000000 26 61.53846154 27 62.96296296 28 60.71428571 29 58.62068966 30 56.66666667 31 54.83870968 32 56.25000000 33 54.54545455 34 52.94117647 35 51.42857143 36 50.00000000 37 51.35135135 38 52.63157895 39 53.84615385 40 55.00000000 41 53.65853659 42 54.76190476 43 53.48837209 44 52.27272727 45 51.11111111 46 52.17391304 47 53.19148936 48 52.08333333 49 53.06122449 50 52.00000000 51 52.94117647 52 53.84615385 53 54.71698113 54 53.70370370 55 52.72727273 56 53.57142857 57 54.38596491 58 55.17241379 59 54.23728814 60 55.00000000 61 55.73770492 62 54.83870968 63 55.55555556 64 56.25000000 65 55.38461538 66 56.06060606 67 56.71641791 68 55.88235294 69 56.52173913 70 55.71428571 71 54.92957746 72 54.16666667 73 54.79452055 74 54.05405405 75 54.66666667 76 55.26315789 77 55.84415584 78 56.41025641 79 55.69620253 80 56.25000000 81 55.55555556 82 56.09756098 83 55.42168675 84 55.95238095 85 56.47058824 86 56.97674419 87 56.32183908 88 56.81818182 89 57.30337079 90 57.77777778 91 58.24175824 92 58.69565217 93 58.06451613 94 57.44680851 95 57.89473684 96 58.33333333 97 57.73195876 98 58.16326531 99 58.58585859 100 59.00000000 101 58.41584158 102 57.84313725 103 57.28155340 104 56.73076923 105 57.14285714 106 56.60377358 107 57.00934579 108 56.48148148 109 56.88073394 110 57.27272727 111 57.65765766 112 58.03571429 113 58.40707965 114 58.77192982 115 59.13043478 116 58.62068966 117 58.97435897 118 59.32203390 119 58.82352941 120 59.16666667 121 58.67768595 122 59.01639344 123 59.34959350 124 58.87096774 125 59.20000000 126 59.52380952 127 59.84251969 128 60.15625000 129 59.68992248 130 59.23076923 131 59.54198473 132 59.84848485 133 60.15037594 134 59.70149254 135 59.25925926 136 58.82352941 137 59.12408759 138 59.42028986 139 59.71223022 140 60.00000000 141 59.57446809 142 59.85915493 143 60.13986014 144 60.41666667 145 60.68965517 146 60.27397260 147 59.86394558 148 59.45945946 149 59.73154362 150 60.00000000 151 59.60264901 152 59.86842105 153 60.13071895 154 59.74025974 155 59.35483871 156 59.61538462 157 59.87261146 158 59.49367089 159 59.74842767 160 59.37500000 161 59.62732919 162 59.87654321 163 60.12269939 164 59.75609756 165 59.39393939 166 59.63855422 167 59.28143713 168 59.52380952 169 59.17159763 170 58.82352941 171 59.06432749 172 59.30232558 173 59.53757225 174 59.77011494 175 59.42857143 176 59.65909091 177 59.88700565 178 59.55056180 179 59.21787709 180 59.44444444 181 59.11602210 182 59.34065934 183 59.01639344 184 58.69565217 185 58.91891892 186 59.13978495 187 59.35828877 188 59.04255319 189 58.73015873 190 58.42105263 191 58.63874346 192 58.85416667 193 58.54922280 194 58.76288660 195 58.97435897 196 58.67346939 197 58.88324873 198 59.09090909 199 58.79396985 200 59.00000000 201 59.20398010 202 59.40594059 203 59.60591133 204 59.80392157 205 59.51219512 206 59.22330097 207 59.42028986 208 59.61538462 209 59.80861244 210 60.00000000 211 60.18957346 212 59.90566038 213 60.09389671 214 60.28037383 215 60.46511628 216 60.64814815 217 60.82949309 218 60.55045872 219 60.73059361 220 60.45454545 221 60.63348416 222 60.81081081 223 60.53811659 224 60.26785714 225 60.00000000 226 60.17699115 227 60.35242291 228 60.52631579 229 60.69868996 230 60.86956522 231 60.60606061 232 60.34482759 233 60.51502146 234 60.25641026 235 60.00000000 236 59.74576271 237 59.91561181 238 60.08403361 239 60.25104603 240 60.41666667 241 60.58091286 242 60.33057851 243 60.49382716 244 60.24590164 245 60.40816327 246 60.56910569 247 60.72874494 248 60.88709677 249 60.64257028 250 60.80000000 251 60.95617530 252 61.11111111 253 61.26482213 254 61.02362205 255 61.17647059 256 61.32812500 257 61.47859922 258 61.62790698 259 61.38996139 260 61.53846154 261 61.30268199 262 61.45038168 263 61.59695817 264 61.74242424 265 61.50943396 266 61.27819549 267 61.42322097 268 61.19402985 269 61.33828996 270 61.48148148 271 61.25461255 272 61.02941176 273 60.80586081 274 60.58394161 275 60.36363636 276 60.50724638 277 60.64981949 278 60.43165468 279 60.57347670 280 60.71428571 281 60.85409253 282 60.99290780 283 60.77738516 284 60.91549296 285 61.05263158 286 61.18881119 287 61.32404181 288 61.45833333 289 61.24567474 290 61.37931034 291 61.16838488 292 61.30136986 293 61.43344710 294 61.22448980 295 61.35593220 296 61.48648649 297 61.61616162 298 61.74496644 299 61.87290970 300 62.00000000 301 61.79401993 302 61.92052980 303 62.04620462 304 62.17105263 305 62.29508197 306 62.09150327 307 61.88925081 308 62.01298701 309 62.13592233 310 62.25806452 311 62.37942122 312 62.17948718 313 62.30031949 314 62.10191083 315 62.22222222 316 62.34177215 317 62.46056782 318 62.57861635 319 62.38244514 320 62.18750000 321 62.30529595 322 62.11180124 323 62.22910217 324 62.03703704 325 62.15384615 326 62.26993865 327 62.07951070 328 62.19512195 329 62.31003040 330 62.12121212 331 62.23564955 332 62.34939759 333 62.46246246 334 62.27544910 335 62.08955224 336 61.90476190 337 61.72106825 338 61.53846154 339 61.65191740 340 61.76470588 341 61.58357771 342 61.40350877 343 61.51603499 344 61.62790698 345 61.44927536 346 61.27167630 347 61.09510086 348 60.91954023 349 61.03151862 350 61.14285714 351 61.25356125 352 61.07954545 353 61.18980170 354 61.01694915 355 60.84507042 356 60.95505618 357 60.78431373 358 60.89385475 359 61.00278552 360 61.11111111 361 61.21883657 362 61.32596685 363 61.15702479 364 60.98901099 365 61.09589041 366 61.20218579 367 61.30790191 368 61.41304348 369 61.51761518 370 61.62162162 371 61.72506739 372 61.82795699 373 61.66219839 374 61.76470588 375 61.60000000 376 61.70212766 377 61.53846154 378 61.64021164 379 61.47757256 380 61.57894737 381 61.67979003 382 61.51832461 383 61.35770235 384 61.19791667 385 61.03896104 386 61.13989637 387 60.98191214 388 60.82474227 389 60.66838046 390 60.51282051 391 60.35805627 392 60.45918367 393 60.55979644 394 60.65989848 395 60.75949367 396 60.60606061 397 60.45340050 398 60.30150754 399 60.40100251 400 60.50000000 401 60.34912718 402 60.44776119 403 60.54590571 404 60.64356436 405 60.74074074 406 60.83743842 407 60.93366093 408 61.02941176 409 60.88019560 410 60.97560976 411 61.07055961 412 61.16504854 413 61.01694915 414 61.11111111 415 60.96385542 416 61.05769231 417 60.91127098 418 60.76555024 419 60.62052506 420 60.71428571 421 60.57007126 422 60.42654028 423 60.52009456 424 60.37735849 425 60.47058824 426 60.56338028 427 60.65573770 428 60.51401869 429 60.60606061 430 60.69767442 431 60.78886311 432 60.87962963 433 60.73903002 434 60.82949309 435 60.91954023 436 61.00917431 437 61.09839817 438 60.95890411 439 61.04783599 440 60.90909091 441 60.99773243 442 60.85972851 443 60.72234763 444 60.58558559 445 60.67415730 446 60.76233184 447 60.85011186 448 60.93750000 449 61.02449889 450 60.88888889 451 60.97560976 452 61.06194690 453 61.14790287 454 61.01321586 455 60.87912088 456 60.96491228 457 61.05032823 458 61.13537118 459 61.22004357 460 61.30434783 461 61.17136659 462 61.25541126 463 61.12311015 464 60.99137931 465 61.07526882 466 61.15879828 467 61.24197002 468 61.32478632 469 61.19402985 470 61.27659574 471 61.35881104 472 61.44067797 473 61.31078224 474 61.39240506 475 61.47368421 476 61.34453782 477 61.42557652 478 61.50627615 479 61.58663883 480 61.66666667 481 61.53846154 482 61.61825726 483 61.69772257 484 61.77685950 485 61.85567010 486 61.93415638 487 61.80698152 488 61.88524590 489 61.75869121 490 61.63265306 491 61.50712831 492 61.58536585 493 61.46044625 494 61.53846154 495 61.41414141 496 61.49193548 497 61.56941650 498 61.64658635 499 61.72344689 500 61.80000000 501 61.67664671 502 61.75298805 503 61.82902584 504 61.70634921 505 61.78217822 506 61.66007905 507 61.53846154 508 61.41732283 509 61.29666012 510 61.37254902 511 61.25244618 512 61.13281250 513 61.01364522 514 61.08949416 515 60.97087379 516 61.04651163 517 61.12185687 518 61.19691120 519 61.27167630 520 61.15384615 521 61.22840691 522 61.30268199 523 61.18546845 524 61.25954198 525 61.14285714 526 61.02661597 527 60.91081594 528 60.79545455 529 60.68052930 530 60.75471698 531 60.82862524 532 60.90225564 533 60.97560976 534 60.86142322 535 60.93457944 536 60.82089552 537 60.89385475 538 60.78066914 539 60.85343228 540 60.74074074 541 60.81330869 542 60.70110701 543 60.58931860 544 60.66176471 545 60.73394495 546 60.62271062 547 60.51188300 548 60.58394161 549 60.65573770 550 60.72727273 551 60.61705989 552 60.50724638 553 60.57866184 554 60.64981949 555 60.72072072 556 60.61151079 557 60.68222621 558 60.57347670 559 60.64400716 560 60.71428571 561 60.78431373 562 60.85409253 563 60.92362345 564 60.81560284 565 60.70796460 566 60.77738516 567 60.67019400 568 60.73943662 569 60.63268893 570 60.52631579 571 60.59544658 572 60.48951049 573 60.55846422 574 60.62717770 575 60.52173913 576 60.59027778 577 60.65857886 578 60.72664360 579 60.79447323 580 60.68965517 581 60.75731497 582 60.65292096 583 60.54888508 584 60.61643836 585 60.68376068 586 60.58020478 587 60.64735945 588 60.71428571 589 60.78098472 590 60.84745763 591 60.91370558 592 60.97972973 593 61.04553120 594 61.11111111 595 61.17647059 596 61.07382550 597 60.97152429 598 61.03678930 599 61.10183639 600 61.00000000 601 61.06489185 602 60.96345515 603 60.86235489 604 60.92715232 605 60.99173554 606 60.89108911 607 60.95551895 608 61.01973684 609 60.91954023 610 60.98360656 611 60.88379705 612 60.94771242 613 60.84828711 614 60.74918567 615 60.65040650 616 60.55194805 617 60.61588331 618 60.51779935 619 60.42003231 620 60.48387097 621 60.54750403 622 60.45016077 623 60.51364366 624 60.41666667 625 60.32000000 626 60.22364217 627 60.28708134 628 60.35031847 629 60.25437202 630 60.31746032 631 60.38034865 632 60.28481013 633 60.18957346 634 60.25236593 635 60.31496063 636 60.37735849 637 60.28257457 638 60.34482759 639 60.40688576 640 60.31250000 641 60.37441498 642 60.28037383 643 60.34214619 644 60.40372671 645 60.31007752 646 60.37151703 647 60.43276662 648 60.49382716 649 60.40061633 650 60.46153846 651 60.52227343 652 60.58282209 653 60.49004594 654 60.55045872 655 60.45801527 656 60.36585366 657 60.27397260 658 60.18237082 659 60.09104704 660 60.15151515 661 60.06051437 662 60.12084592 663 60.18099548 664 60.24096386 665 60.30075188 666 60.21021021 667 60.26986507 668 60.17964072 669 60.08968610 670 60.14925373 671 60.05961252 672 60.11904762 673 60.02971768 674 59.94065282 675 59.85185185 676 59.76331361 677 59.82274742 678 59.73451327 679 59.79381443 680 59.85294118 681 59.76505140 682 59.67741935 683 59.59004392 684 59.50292398 685 59.56204380 686 59.47521866 687 59.38864629 688 59.44767442 689 59.36139332 690 59.42028986 691 59.47901592 692 59.53757225 693 59.59595960 694 59.51008646 695 59.42446043 696 59.33908046 697 59.25394548 698 59.16905444 699 59.08440629 700 59.14285714 701 59.05848787 702 59.11680912 703 59.17496444 704 59.23295455 705 59.14893617 706 59.20679887 707 59.12305516 708 59.18079096 709 59.23836389 710 59.29577465 711 59.35302391 712 59.41011236 713 59.46704067 714 59.38375350 715 59.30069930 716 59.35754190 717 59.27475593 718 59.19220056 719 59.10987483 720 59.16666667 721 59.22330097 722 59.27977839 723 59.33609959 724 59.25414365 725 59.31034483 726 59.22865014 727 59.14718019 728 59.20329670 729 59.25925926 730 59.31506849 731 59.37072503 732 59.42622951 733 59.48158254 734 59.53678474 735 59.45578231 736 59.51086957 737 59.43012212 738 59.48509485 739 59.40460081 740 59.32432432 741 59.37921727 742 59.43396226 743 59.48855989 744 59.40860215 745 59.46308725 746 59.51742627 747 59.43775100 748 59.35828877 749 59.27903872 750 59.33333333 Final result: 59.3333 +/- 1.7948 Random chance: 25.0083 +/- 1.5824