| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 1548 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 1548 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 50.00000000 | |
| 5 40.00000000 | |
| 6 33.33333333 | |
| 7 42.85714286 | |
| 8 37.50000000 | |
| 9 33.33333333 | |
| 10 30.00000000 | |
| 11 27.27272727 | |
| 12 25.00000000 | |
| 13 30.76923077 | |
| 14 28.57142857 | |
| 15 33.33333333 | |
| 16 37.50000000 | |
| 17 35.29411765 | |
| 18 33.33333333 | |
| 19 36.84210526 | |
| 20 35.00000000 | |
| 21 33.33333333 | |
| 22 31.81818182 | |
| 23 30.43478261 | |
| 24 29.16666667 | |
| 25 32.00000000 | |
| 26 30.76923077 | |
| 27 29.62962963 | |
| 28 32.14285714 | |
| 29 34.48275862 | |
| 30 36.66666667 | |
| 31 38.70967742 | |
| 32 40.62500000 | |
| 33 39.39393939 | |
| 34 38.23529412 | |
| 35 40.00000000 | |
| 36 38.88888889 | |
| 37 37.83783784 | |
| 38 36.84210526 | |
| 39 35.89743590 | |
| 40 35.00000000 | |
| 41 34.14634146 | |
| 42 33.33333333 | |
| 43 32.55813953 | |
| 44 31.81818182 | |
| 45 31.11111111 | |
| 46 30.43478261 | |
| 47 29.78723404 | |
| 48 31.25000000 | |
| 49 32.65306122 | |
| 50 34.00000000 | |
| 51 33.33333333 | |
| 52 32.69230769 | |
| 53 32.07547170 | |
| 54 31.48148148 | |
| 55 32.72727273 | |
| 56 32.14285714 | |
| 57 33.33333333 | |
| 58 32.75862069 | |
| 59 32.20338983 | |
| 60 31.66666667 | |
| 61 31.14754098 | |
| 62 30.64516129 | |
| 63 30.15873016 | |
| 64 29.68750000 | |
| 65 30.76923077 | |
| 66 31.81818182 | |
| 67 31.34328358 | |
| 68 30.88235294 | |
| 69 30.43478261 | |
| 70 31.42857143 | |
| 71 30.98591549 | |
| 72 31.94444444 | |
| 73 31.50684932 | |
| 74 31.08108108 | |
| 75 32.00000000 | |
| 76 32.89473684 | |
| 77 33.76623377 | |
| 78 34.61538462 | |
| 79 34.17721519 | |
| 80 35.00000000 | |
| 81 35.80246914 | |
| 82 35.36585366 | |
| 83 36.14457831 | |
| 84 36.90476190 | |
| 85 36.47058824 | |
| 86 36.04651163 | |
| 87 36.78160920 | |
| 88 37.50000000 | |
| 89 37.07865169 | |
| 90 36.66666667 | |
| 91 36.26373626 | |
| 92 35.86956522 | |
| 93 35.48387097 | |
| 94 36.17021277 | |
| 95 35.78947368 | |
| 96 36.45833333 | |
| 97 36.08247423 | |
| 98 35.71428571 | |
| 99 35.35353535 | |
| 100 36.00000000 | |
| 101 35.64356436 | |
| 102 35.29411765 | |
| 103 34.95145631 | |
| 104 34.61538462 | |
| 105 34.28571429 | |
| 106 33.96226415 | |
| 107 33.64485981 | |
| 108 33.33333333 | |
| 109 33.02752294 | |
| 110 32.72727273 | |
| 111 32.43243243 | |
| 112 32.14285714 | |
| 113 31.85840708 | |
| 114 32.45614035 | |
| 115 33.04347826 | |
| 116 32.75862069 | |
| 117 32.47863248 | |
| 118 32.20338983 | |
| 119 31.93277311 | |
| 120 31.66666667 | |
| 121 31.40495868 | |
| 122 31.14754098 | |
| 123 30.89430894 | |
| 124 30.64516129 | |
| 125 31.20000000 | |
| 126 30.95238095 | |
| 127 30.70866142 | |
| 128 30.46875000 | |
| 129 30.23255814 | |
| 130 30.00000000 | |
| 131 30.53435115 | |
| 132 31.06060606 | |
| 133 30.82706767 | |
| 134 30.59701493 | |
| 135 30.37037037 | |
| 136 30.14705882 | |
| 137 30.65693431 | |
| 138 30.43478261 | |
| 139 30.93525180 | |
| 140 30.71428571 | |
| 141 31.20567376 | |
| 142 30.98591549 | |
| 143 31.46853147 | |
| 144 31.94444444 | |
| 145 31.72413793 | |
| 146 31.50684932 | |
| 147 31.29251701 | |
| 148 31.75675676 | |
| 149 31.54362416 | |
| 150 31.33333333 | |
| 151 31.12582781 | |
| 152 31.57894737 | |
| 153 31.37254902 | |
| 154 31.16883117 | |
| 155 30.96774194 | |
| 156 30.76923077 | |
| 157 31.21019108 | |
| 158 31.01265823 | |
| 159 30.81761006 | |
| 160 31.25000000 | |
| 161 31.05590062 | |
| 162 30.86419753 | |
| 163 30.67484663 | |
| 164 31.09756098 | |
| 165 31.51515152 | |
| 166 31.32530120 | |
| 167 31.73652695 | |
| 168 31.54761905 | |
| 169 31.36094675 | |
| 170 31.17647059 | |
| 171 30.99415205 | |
| 172 30.81395349 | |
| 173 30.63583815 | |
| 174 30.45977011 | |
| 175 30.28571429 | |
| 176 30.11363636 | |
| 177 29.94350282 | |
| 178 29.77528090 | |
| 179 30.16759777 | |
| 180 30.00000000 | |
| 181 29.83425414 | |
| 182 29.67032967 | |
| 183 30.05464481 | |
| 184 29.89130435 | |
| 185 29.72972973 | |
| 186 29.56989247 | |
| 187 29.41176471 | |
| 188 29.25531915 | |
| 189 29.10052910 | |
| 190 28.94736842 | |
| 191 29.31937173 | |
| 192 29.16666667 | |
| 193 29.01554404 | |
| 194 29.38144330 | |
| 195 29.74358974 | |
| 196 29.59183673 | |
| 197 29.44162437 | |
| 198 29.29292929 | |
| 199 29.14572864 | |
| 200 29.00000000 | |
| 201 29.35323383 | |
| 202 29.20792079 | |
| 203 29.06403941 | |
| 204 28.92156863 | |
| 205 29.26829268 | |
| 206 29.12621359 | |
| 207 28.98550725 | |
| 208 28.84615385 | |
| 209 28.70813397 | |
| 210 28.57142857 | |
| 211 28.90995261 | |
| 212 28.77358491 | |
| 213 28.63849765 | |
| 214 28.50467290 | |
| 215 28.37209302 | |
| 216 28.24074074 | |
| 217 28.11059908 | |
| 218 27.98165138 | |
| 219 27.85388128 | |
| 220 27.72727273 | |
| 221 27.60180995 | |
| 222 27.92792793 | |
| 223 27.80269058 | |
| 224 27.67857143 | |
| 225 28.00000000 | |
| 226 27.87610619 | |
| 227 28.19383260 | |
| 228 28.50877193 | |
| 229 28.38427948 | |
| 230 28.26086957 | |
| 231 28.57142857 | |
| 232 28.44827586 | |
| 233 28.32618026 | |
| 234 28.63247863 | |
| 235 28.51063830 | |
| 236 28.38983051 | |
| 237 28.27004219 | |
| 238 28.15126050 | |
| 239 28.03347280 | |
| 240 28.33333333 | |
| 241 28.21576763 | |
| 242 28.51239669 | |
| 243 28.80658436 | |
| 244 28.68852459 | |
| 245 28.97959184 | |
| 246 28.86178862 | |
| 247 28.74493927 | |
| 248 28.62903226 | |
| 249 28.91566265 | |
| 250 28.80000000 | |
| 251 28.68525896 | |
| 252 28.57142857 | |
| 253 28.45849802 | |
| 254 28.34645669 | |
| 255 28.62745098 | |
| 256 28.51562500 | |
| 257 28.40466926 | |
| 258 28.29457364 | |
| 259 28.18532819 | |
| 260 28.07692308 | |
| 261 27.96934866 | |
| 262 27.86259542 | |
| 263 27.75665399 | |
| 264 27.65151515 | |
| 265 27.54716981 | |
| 266 27.81954887 | |
| 267 27.71535581 | |
| 268 27.61194030 | |
| 269 27.50929368 | |
| 270 27.40740741 | |
| 271 27.30627306 | |
| 272 27.57352941 | |
| 273 27.47252747 | |
| 274 27.37226277 | |
| 275 27.63636364 | |
| 276 27.89855072 | |
| 277 28.15884477 | |
| 278 28.41726619 | |
| 279 28.31541219 | |
| 280 28.57142857 | |
| 281 28.46975089 | |
| 282 28.36879433 | |
| 283 28.26855124 | |
| 284 28.16901408 | |
| 285 28.07017544 | |
| 286 27.97202797 | |
| 287 28.22299652 | |
| 288 28.12500000 | |
| 289 28.02768166 | |
| 290 28.27586207 | |
| 291 28.52233677 | |
| 292 28.76712329 | |
| 293 29.01023891 | |
| 294 29.25170068 | |
| 295 29.15254237 | |
| 296 29.05405405 | |
| 297 28.95622896 | |
| 298 28.85906040 | |
| 299 29.09698997 | |
| 300 29.33333333 | |
| 301 29.56810631 | |
| 302 29.80132450 | |
| 303 29.70297030 | |
| 304 29.60526316 | |
| 305 29.83606557 | |
| 306 29.73856209 | |
| 307 29.64169381 | |
| 308 29.54545455 | |
| 309 29.44983819 | |
| 310 29.35483871 | |
| 311 29.26045016 | |
| 312 29.48717949 | |
| 313 29.71246006 | |
| 314 29.93630573 | |
| 315 29.84126984 | |
| 316 29.74683544 | |
| 317 29.65299685 | |
| 318 29.55974843 | |
| 319 29.78056426 | |
| 320 29.68750000 | |
| 321 29.59501558 | |
| 322 29.50310559 | |
| 323 29.41176471 | |
| 324 29.32098765 | |
| 325 29.53846154 | |
| 326 29.44785276 | |
| 327 29.66360856 | |
| 328 29.87804878 | |
| 329 29.78723404 | |
| 330 29.69696970 | |
| 331 29.60725076 | |
| 332 29.81927711 | |
| 333 29.72972973 | |
| 334 29.64071856 | |
| 335 29.85074627 | |
| 336 29.76190476 | |
| 337 29.67359050 | |
| 338 29.58579882 | |
| 339 29.49852507 | |
| 340 29.70588235 | |
| 341 29.61876833 | |
| 342 29.53216374 | |
| 343 29.44606414 | |
| 344 29.36046512 | |
| 345 29.27536232 | |
| 346 29.47976879 | |
| 347 29.68299712 | |
| 348 29.59770115 | |
| 349 29.79942693 | |
| 350 30.00000000 | |
| 351 29.91452991 | |
| 352 29.82954545 | |
| 353 30.02832861 | |
| 354 29.94350282 | |
| 355 29.85915493 | |
| 356 30.05617978 | |
| 357 29.97198880 | |
| 358 29.88826816 | |
| 359 29.80501393 | |
| 360 30.00000000 | |
| 361 30.19390582 | |
| 362 30.38674033 | |
| 363 30.57851240 | |
| 364 30.49450549 | |
| 365 30.41095890 | |
| 366 30.32786885 | |
| 367 30.24523161 | |
| 368 30.16304348 | |
| 369 30.08130081 | |
| 370 30.27027027 | |
| 371 30.45822102 | |
| 372 30.37634409 | |
| 373 30.29490617 | |
| 374 30.21390374 | |
| 375 30.40000000 | |
| 376 30.31914894 | |
| 377 30.23872679 | |
| 378 30.42328042 | |
| 379 30.34300792 | |
| 380 30.26315789 | |
| 381 30.44619423 | |
| 382 30.36649215 | |
| 383 30.54830287 | |
| 384 30.46875000 | |
| 385 30.64935065 | |
| 386 30.56994819 | |
| 387 30.49095607 | |
| 388 30.41237113 | |
| 389 30.59125964 | |
| 390 30.51282051 | |
| 391 30.69053708 | |
| 392 30.86734694 | |
| 393 30.78880407 | |
| 394 30.71065990 | |
| 395 30.63291139 | |
| 396 30.80808081 | |
| 397 30.98236776 | |
| 398 30.90452261 | |
| 399 31.07769424 | |
| 400 31.00000000 | |
| 401 31.17206983 | |
| 402 31.34328358 | |
| 403 31.26550868 | |
| 404 31.18811881 | |
| 405 31.11111111 | |
| 406 31.03448276 | |
| 407 30.95823096 | |
| 408 30.88235294 | |
| 409 31.05134474 | |
| 410 30.97560976 | |
| 411 30.90024331 | |
| 412 30.82524272 | |
| 413 30.99273608 | |
| 414 31.15942029 | |
| 415 31.32530120 | |
| 416 31.25000000 | |
| 417 31.17505995 | |
| 418 31.10047847 | |
| 419 31.26491647 | |
| 420 31.42857143 | |
| 421 31.35391924 | |
| 422 31.27962085 | |
| 423 31.20567376 | |
| 424 31.36792453 | |
| 425 31.52941176 | |
| 426 31.45539906 | |
| 427 31.38173302 | |
| 428 31.30841121 | |
| 429 31.46853147 | |
| 430 31.39534884 | |
| 431 31.55452436 | |
| 432 31.71296296 | |
| 433 31.63972286 | |
| 434 31.56682028 | |
| 435 31.49425287 | |
| 436 31.42201835 | |
| 437 31.35011442 | |
| 438 31.50684932 | |
| 439 31.66287016 | |
| 440 31.59090909 | |
| 441 31.51927438 | |
| 442 31.44796380 | |
| 443 31.37697517 | |
| 444 31.53153153 | |
| 445 31.46067416 | |
| 446 31.39013453 | |
| 447 31.31991051 | |
| 448 31.25000000 | |
| 449 31.18040089 | |
| 450 31.11111111 | |
| 451 31.04212860 | |
| 452 31.19469027 | |
| 453 31.34657837 | |
| 454 31.49779736 | |
| 455 31.42857143 | |
| 456 31.57894737 | |
| 457 31.50984683 | |
| 458 31.44104803 | |
| 459 31.59041394 | |
| 460 31.73913043 | |
| 461 31.67028200 | |
| 462 31.60173160 | |
| 463 31.53347732 | |
| 464 31.46551724 | |
| 465 31.39784946 | |
| 466 31.33047210 | |
| 467 31.47751606 | |
| 468 31.41025641 | |
| 469 31.55650320 | |
| 470 31.48936170 | |
| 471 31.42250531 | |
| 472 31.35593220 | |
| 473 31.50105708 | |
| 474 31.43459916 | |
| 475 31.36842105 | |
| 476 31.51260504 | |
| 477 31.44654088 | |
| 478 31.58995816 | |
| 479 31.73277662 | |
| 480 31.66666667 | |
| 481 31.80873181 | |
| 482 31.74273859 | |
| 483 31.67701863 | |
| 484 31.61157025 | |
| 485 31.54639175 | |
| 486 31.48148148 | |
| 487 31.62217659 | |
| 488 31.55737705 | |
| 489 31.49284254 | |
| 490 31.63265306 | |
| 491 31.77189409 | |
| 492 31.70731707 | |
| 493 31.64300203 | |
| 494 31.78137652 | |
| 495 31.91919192 | |
| 496 32.05645161 | |
| 497 31.99195171 | |
| 498 31.92771084 | |
| 499 31.86372745 | |
| 500 32.00000000 | |
| 501 31.93612774 | |
| 502 31.87250996 | |
| 503 31.80914513 | |
| 504 31.94444444 | |
| 505 31.88118812 | |
| 506 32.01581028 | |
| 507 31.95266272 | |
| 508 31.88976378 | |
| 509 31.82711198 | |
| 510 31.96078431 | |
| 511 31.89823875 | |
| 512 31.83593750 | |
| 513 31.77387914 | |
| 514 31.71206226 | |
| 515 31.65048544 | |
| 516 31.58914729 | |
| 517 31.72147002 | |
| 518 31.85328185 | |
| 519 31.79190751 | |
| 520 31.73076923 | |
| 521 31.66986564 | |
| 522 31.60919540 | |
| 523 31.73996176 | |
| 524 31.87022901 | |
| 525 31.80952381 | |
| 526 31.74904943 | |
| 527 31.87855787 | |
| 528 31.81818182 | |
| 529 31.94706994 | |
| 530 32.07547170 | |
| 531 32.01506591 | |
| 532 31.95488722 | |
| 533 31.89493433 | |
| 534 31.83520599 | |
| 535 31.96261682 | |
| 536 31.90298507 | |
| 537 32.02979516 | |
| 538 32.15613383 | |
| 539 32.09647495 | |
| 540 32.03703704 | |
| 541 31.97781885 | |
| 542 32.10332103 | |
| 543 32.04419890 | |
| 544 31.98529412 | |
| 545 31.92660550 | |
| 546 31.86813187 | |
| 547 31.80987203 | |
| 548 31.93430657 | |
| 549 31.87613843 | |
| 550 31.81818182 | |
| 551 31.94192377 | |
| 552 31.88405797 | |
| 553 31.82640145 | |
| 554 31.76895307 | |
| 555 31.71171171 | |
| 556 31.83453237 | |
| 557 31.77737882 | |
| 558 31.89964158 | |
| 559 31.84257603 | |
| 560 31.78571429 | |
| 561 31.72905526 | |
| 562 31.67259786 | |
| 563 31.79396092 | |
| 564 31.73758865 | |
| 565 31.85840708 | |
| 566 31.80212014 | |
| 567 31.74603175 | |
| 568 31.86619718 | |
| 569 31.81019332 | |
| 570 31.75438596 | |
| 571 31.69877408 | |
| 572 31.64335664 | |
| 573 31.58813264 | |
| 574 31.70731707 | |
| 575 31.65217391 | |
| 576 31.77083333 | |
| 577 31.88908146 | |
| 578 31.83391003 | |
| 579 31.95164076 | |
| 580 31.89655172 | |
| 581 31.84165232 | |
| 582 31.78694158 | |
| 583 31.90394511 | |
| 584 32.02054795 | |
| 585 31.96581197 | |
| 586 32.08191126 | |
| 587 32.02725724 | |
| 588 31.97278912 | |
| 589 31.91850594 | |
| 590 31.86440678 | |
| 591 31.81049069 | |
| 592 31.75675676 | |
| 593 31.70320405 | |
| 594 31.81818182 | |
| 595 31.76470588 | |
| 596 31.87919463 | |
| 597 31.82579564 | |
| 598 31.77257525 | |
| 599 31.88647746 | |
| 600 31.83333333 | |
| 601 31.94675541 | |
| 602 31.89368771 | |
| 603 32.00663350 | |
| 604 31.95364238 | |
| 605 32.06611570 | |
| 606 32.01320132 | |
| 607 32.12520593 | |
| 608 32.07236842 | |
| 609 32.01970443 | |
| 610 31.96721311 | |
| 611 31.91489362 | |
| 612 31.86274510 | |
| 613 31.97389886 | |
| 614 32.08469055 | |
| 615 32.03252033 | |
| 616 32.14285714 | |
| 617 32.09076175 | |
| 618 32.03883495 | |
| 619 31.98707593 | |
| 620 31.93548387 | |
| 621 31.88405797 | |
| 622 31.83279743 | |
| 623 31.78170144 | |
| 624 31.73076923 | |
| 625 31.68000000 | |
| 626 31.78913738 | |
| 627 31.73843700 | |
| 628 31.68789809 | |
| 629 31.79650238 | |
| 630 31.90476190 | |
| 631 31.85419968 | |
| 632 31.96202532 | |
| 633 31.91153239 | |
| 634 31.86119874 | |
| 635 31.81102362 | |
| 636 31.76100629 | |
| 637 31.71114600 | |
| 638 31.66144201 | |
| 639 31.76838811 | |
| 640 31.87500000 | |
| 641 31.82527301 | |
| 642 31.93146417 | |
| 643 31.88180404 | |
| 644 31.98757764 | |
| 645 31.93798450 | |
| 646 31.88854489 | |
| 647 31.83925811 | |
| 648 31.79012346 | |
| 649 31.74114022 | |
| 650 31.69230769 | |
| 651 31.64362519 | |
| 652 31.74846626 | |
| 653 31.69984686 | |
| 654 31.80428135 | |
| 655 31.75572519 | |
| 656 31.85975610 | |
| 657 31.81126332 | |
| 658 31.91489362 | |
| 659 31.86646434 | |
| 660 31.81818182 | |
| 661 31.77004539 | |
| 662 31.72205438 | |
| 663 31.67420814 | |
| 664 31.77710843 | |
| 665 31.72932331 | |
| 666 31.68168168 | |
| 667 31.63418291 | |
| 668 31.73652695 | |
| 669 31.68908819 | |
| 670 31.64179104 | |
| 671 31.59463487 | |
| 672 31.54761905 | |
| 673 31.50074294 | |
| 674 31.45400593 | |
| 675 31.40740741 | |
| 676 31.36094675 | |
| 677 31.31462334 | |
| 678 31.41592920 | |
| 679 31.36966127 | |
| 680 31.47058824 | |
| 681 31.57121880 | |
| 682 31.52492669 | |
| 683 31.62518302 | |
| 684 31.57894737 | |
| 685 31.53284672 | |
| 686 31.48688047 | |
| 687 31.44104803 | |
| 688 31.39534884 | |
| 689 31.34978229 | |
| 690 31.30434783 | |
| 691 31.25904486 | |
| 692 31.21387283 | |
| 693 31.16883117 | |
| 694 31.26801153 | |
| 695 31.22302158 | |
| 696 31.32183908 | |
| 697 31.27690100 | |
| 698 31.23209169 | |
| 699 31.33047210 | |
| 700 31.28571429 | |
| 701 31.38373752 | |
| 702 31.48148148 | |
| 703 31.57894737 | |
| 704 31.53409091 | |
| 705 31.48936170 | |
| 706 31.44475921 | |
| 707 31.40028289 | |
| 708 31.35593220 | |
| 709 31.31170663 | |
| 710 31.40845070 | |
| 711 31.50492264 | |
| 712 31.46067416 | |
| 713 31.55680224 | |
| 714 31.65266106 | |
| 715 31.60839161 | |
| 716 31.56424581 | |
| 717 31.52022315 | |
| 718 31.47632312 | |
| 719 31.43254520 | |
| 720 31.38888889 | |
| 721 31.34535368 | |
| 722 31.30193906 | |
| 723 31.39695712 | |
| 724 31.35359116 | |
| 725 31.44827586 | |
| 726 31.54269972 | |
| 727 31.49931224 | |
| 728 31.45604396 | |
| 729 31.41289438 | |
| 730 31.36986301 | |
| 731 31.32694938 | |
| 732 31.42076503 | |
| 733 31.51432469 | |
| 734 31.60762943 | |
| 735 31.56462585 | |
| 736 31.52173913 | |
| 737 31.61465400 | |
| 738 31.57181572 | |
| 739 31.66441137 | |
| 740 31.62162162 | |
| 741 31.57894737 | |
| 742 31.67115903 | |
| 743 31.76312248 | |
| 744 31.85483871 | |
| 745 31.81208054 | |
| 746 31.76943700 | |
| 747 31.72690763 | |
| 748 31.68449198 | |
| 749 31.64218959 | |
| 750 31.73333333 | |
| Final result: 31.7333 ±1.7007 | |
| Random chance: 25.0000 ±1.5822 | |