| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 1548 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 1548 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 50.00000000 | |
| 5 40.00000000 | |
| 6 33.33333333 | |
| 7 42.85714286 | |
| 8 50.00000000 | |
| 9 44.44444444 | |
| 10 40.00000000 | |
| 11 36.36363636 | |
| 12 33.33333333 | |
| 13 38.46153846 | |
| 14 35.71428571 | |
| 15 33.33333333 | |
| 16 37.50000000 | |
| 17 35.29411765 | |
| 18 33.33333333 | |
| 19 36.84210526 | |
| 20 35.00000000 | |
| 21 33.33333333 | |
| 22 31.81818182 | |
| 23 30.43478261 | |
| 24 29.16666667 | |
| 25 32.00000000 | |
| 26 30.76923077 | |
| 27 29.62962963 | |
| 28 32.14285714 | |
| 29 34.48275862 | |
| 30 36.66666667 | |
| 31 38.70967742 | |
| 32 40.62500000 | |
| 33 39.39393939 | |
| 34 38.23529412 | |
| 35 40.00000000 | |
| 36 38.88888889 | |
| 37 37.83783784 | |
| 38 39.47368421 | |
| 39 38.46153846 | |
| 40 37.50000000 | |
| 41 36.58536585 | |
| 42 35.71428571 | |
| 43 34.88372093 | |
| 44 34.09090909 | |
| 45 33.33333333 | |
| 46 32.60869565 | |
| 47 31.91489362 | |
| 48 33.33333333 | |
| 49 34.69387755 | |
| 50 36.00000000 | |
| 51 35.29411765 | |
| 52 34.61538462 | |
| 53 33.96226415 | |
| 54 33.33333333 | |
| 55 34.54545455 | |
| 56 35.71428571 | |
| 57 35.08771930 | |
| 58 34.48275862 | |
| 59 33.89830508 | |
| 60 33.33333333 | |
| 61 32.78688525 | |
| 62 32.25806452 | |
| 63 31.74603175 | |
| 64 31.25000000 | |
| 65 30.76923077 | |
| 66 31.81818182 | |
| 67 31.34328358 | |
| 68 30.88235294 | |
| 69 31.88405797 | |
| 70 32.85714286 | |
| 71 32.39436620 | |
| 72 33.33333333 | |
| 73 32.87671233 | |
| 74 32.43243243 | |
| 75 33.33333333 | |
| 76 34.21052632 | |
| 77 35.06493506 | |
| 78 35.89743590 | |
| 79 35.44303797 | |
| 80 36.25000000 | |
| 81 35.80246914 | |
| 82 35.36585366 | |
| 83 36.14457831 | |
| 84 36.90476190 | |
| 85 36.47058824 | |
| 86 36.04651163 | |
| 87 36.78160920 | |
| 88 37.50000000 | |
| 89 37.07865169 | |
| 90 36.66666667 | |
| 91 36.26373626 | |
| 92 35.86956522 | |
| 93 35.48387097 | |
| 94 35.10638298 | |
| 95 34.73684211 | |
| 96 35.41666667 | |
| 97 35.05154639 | |
| 98 34.69387755 | |
| 99 34.34343434 | |
| 100 34.00000000 | |
| 101 33.66336634 | |
| 102 33.33333333 | |
| 103 33.00970874 | |
| 104 32.69230769 | |
| 105 32.38095238 | |
| 106 32.07547170 | |
| 107 31.77570093 | |
| 108 32.40740741 | |
| 109 32.11009174 | |
| 110 31.81818182 | |
| 111 31.53153153 | |
| 112 31.25000000 | |
| 113 31.85840708 | |
| 114 32.45614035 | |
| 115 33.04347826 | |
| 116 32.75862069 | |
| 117 32.47863248 | |
| 118 32.20338983 | |
| 119 31.93277311 | |
| 120 31.66666667 | |
| 121 31.40495868 | |
| 122 31.14754098 | |
| 123 30.89430894 | |
| 124 30.64516129 | |
| 125 31.20000000 | |
| 126 30.95238095 | |
| 127 30.70866142 | |
| 128 30.46875000 | |
| 129 30.23255814 | |
| 130 30.00000000 | |
| 131 30.53435115 | |
| 132 30.30303030 | |
| 133 30.07518797 | |
| 134 29.85074627 | |
| 135 29.62962963 | |
| 136 30.14705882 | |
| 137 29.92700730 | |
| 138 29.71014493 | |
| 139 29.49640288 | |
| 140 29.28571429 | |
| 141 29.07801418 | |
| 142 29.57746479 | |
| 143 30.06993007 | |
| 144 30.55555556 | |
| 145 31.03448276 | |
| 146 30.82191781 | |
| 147 30.61224490 | |
| 148 31.08108108 | |
| 149 30.87248322 | |
| 150 30.66666667 | |
| 151 30.46357616 | |
| 152 30.92105263 | |
| 153 30.71895425 | |
| 154 31.16883117 | |
| 155 30.96774194 | |
| 156 30.76923077 | |
| 157 31.21019108 | |
| 158 31.64556962 | |
| 159 31.44654088 | |
| 160 31.87500000 | |
| 161 31.67701863 | |
| 162 31.48148148 | |
| 163 31.28834356 | |
| 164 31.09756098 | |
| 165 31.51515152 | |
| 166 31.32530120 | |
| 167 31.73652695 | |
| 168 31.54761905 | |
| 169 31.36094675 | |
| 170 31.17647059 | |
| 171 30.99415205 | |
| 172 30.81395349 | |
| 173 30.63583815 | |
| 174 30.45977011 | |
| 175 30.28571429 | |
| 176 30.68181818 | |
| 177 30.50847458 | |
| 178 30.33707865 | |
| 179 30.72625698 | |
| 180 30.55555556 | |
| 181 30.38674033 | |
| 182 30.21978022 | |
| 183 30.60109290 | |
| 184 30.43478261 | |
| 185 30.81081081 | |
| 186 31.18279570 | |
| 187 31.01604278 | |
| 188 30.85106383 | |
| 189 30.68783069 | |
| 190 30.52631579 | |
| 191 30.36649215 | |
| 192 30.20833333 | |
| 193 30.05181347 | |
| 194 30.41237113 | |
| 195 30.76923077 | |
| 196 31.12244898 | |
| 197 30.96446701 | |
| 198 30.80808081 | |
| 199 30.65326633 | |
| 200 30.50000000 | |
| 201 30.84577114 | |
| 202 30.69306931 | |
| 203 31.03448276 | |
| 204 30.88235294 | |
| 205 31.21951220 | |
| 206 31.06796117 | |
| 207 30.91787440 | |
| 208 30.76923077 | |
| 209 30.62200957 | |
| 210 30.47619048 | |
| 211 30.80568720 | |
| 212 30.66037736 | |
| 213 30.51643192 | |
| 214 30.37383178 | |
| 215 30.23255814 | |
| 216 30.09259259 | |
| 217 29.95391705 | |
| 218 29.81651376 | |
| 219 29.68036530 | |
| 220 29.54545455 | |
| 221 29.41176471 | |
| 222 29.72972973 | |
| 223 29.59641256 | |
| 224 29.91071429 | |
| 225 30.22222222 | |
| 226 30.08849558 | |
| 227 30.39647577 | |
| 228 30.70175439 | |
| 229 30.56768559 | |
| 230 30.43478261 | |
| 231 30.30303030 | |
| 232 30.17241379 | |
| 233 30.04291845 | |
| 234 30.34188034 | |
| 235 30.21276596 | |
| 236 30.08474576 | |
| 237 29.95780591 | |
| 238 29.83193277 | |
| 239 29.70711297 | |
| 240 29.58333333 | |
| 241 29.46058091 | |
| 242 29.75206612 | |
| 243 30.04115226 | |
| 244 29.91803279 | |
| 245 30.20408163 | |
| 246 30.08130081 | |
| 247 29.95951417 | |
| 248 29.83870968 | |
| 249 29.71887550 | |
| 250 29.60000000 | |
| 251 29.48207171 | |
| 252 29.36507937 | |
| 253 29.64426877 | |
| 254 29.52755906 | |
| 255 29.80392157 | |
| 256 29.68750000 | |
| 257 29.96108949 | |
| 258 29.84496124 | |
| 259 30.11583012 | |
| 260 30.00000000 | |
| 261 29.88505747 | |
| 262 29.77099237 | |
| 263 29.65779468 | |
| 264 29.92424242 | |
| 265 30.18867925 | |
| 266 30.07518797 | |
| 267 29.96254682 | |
| 268 29.85074627 | |
| 269 29.73977695 | |
| 270 29.62962963 | |
| 271 29.52029520 | |
| 272 29.77941176 | |
| 273 29.67032967 | |
| 274 29.56204380 | |
| 275 29.45454545 | |
| 276 29.71014493 | |
| 277 29.96389892 | |
| 278 30.21582734 | |
| 279 30.10752688 | |
| 280 30.00000000 | |
| 281 29.89323843 | |
| 282 29.78723404 | |
| 283 29.68197880 | |
| 284 29.57746479 | |
| 285 29.47368421 | |
| 286 29.37062937 | |
| 287 29.61672474 | |
| 288 29.51388889 | |
| 289 29.41176471 | |
| 290 29.65517241 | |
| 291 29.89690722 | |
| 292 30.13698630 | |
| 293 30.37542662 | |
| 294 30.61224490 | |
| 295 30.50847458 | |
| 296 30.40540541 | |
| 297 30.30303030 | |
| 298 30.53691275 | |
| 299 30.43478261 | |
| 300 30.66666667 | |
| 301 30.56478405 | |
| 302 30.79470199 | |
| 303 30.69306931 | |
| 304 30.92105263 | |
| 305 31.14754098 | |
| 306 31.04575163 | |
| 307 31.27035831 | |
| 308 31.16883117 | |
| 309 31.06796117 | |
| 310 30.96774194 | |
| 311 30.86816720 | |
| 312 31.08974359 | |
| 313 31.30990415 | |
| 314 31.52866242 | |
| 315 31.42857143 | |
| 316 31.32911392 | |
| 317 31.23028391 | |
| 318 31.13207547 | |
| 319 31.34796238 | |
| 320 31.25000000 | |
| 321 31.15264798 | |
| 322 31.05590062 | |
| 323 30.95975232 | |
| 324 30.86419753 | |
| 325 31.07692308 | |
| 326 30.98159509 | |
| 327 31.19266055 | |
| 328 31.09756098 | |
| 329 31.00303951 | |
| 330 30.90909091 | |
| 331 30.81570997 | |
| 332 31.02409639 | |
| 333 30.93093093 | |
| 334 30.83832335 | |
| 335 31.04477612 | |
| 336 30.95238095 | |
| 337 30.86053412 | |
| 338 30.76923077 | |
| 339 30.67846608 | |
| 340 30.58823529 | |
| 341 30.49853372 | |
| 342 30.40935673 | |
| 343 30.32069971 | |
| 344 30.23255814 | |
| 345 30.14492754 | |
| 346 30.34682081 | |
| 347 30.54755043 | |
| 348 30.45977011 | |
| 349 30.65902579 | |
| 350 30.85714286 | |
| 351 30.76923077 | |
| 352 30.68181818 | |
| 353 30.87818697 | |
| 354 30.79096045 | |
| 355 30.70422535 | |
| 356 30.89887640 | |
| 357 30.81232493 | |
| 358 30.72625698 | |
| 359 30.91922006 | |
| 360 31.11111111 | |
| 361 31.30193906 | |
| 362 31.21546961 | |
| 363 31.40495868 | |
| 364 31.31868132 | |
| 365 31.23287671 | |
| 366 31.14754098 | |
| 367 31.06267030 | |
| 368 30.97826087 | |
| 369 30.89430894 | |
| 370 31.08108108 | |
| 371 30.99730458 | |
| 372 30.91397849 | |
| 373 30.83109920 | |
| 374 31.01604278 | |
| 375 30.93333333 | |
| 376 30.85106383 | |
| 377 30.76923077 | |
| 378 30.68783069 | |
| 379 30.60686016 | |
| 380 30.52631579 | |
| 381 30.70866142 | |
| 382 30.62827225 | |
| 383 30.80939948 | |
| 384 30.72916667 | |
| 385 30.64935065 | |
| 386 30.56994819 | |
| 387 30.49095607 | |
| 388 30.41237113 | |
| 389 30.59125964 | |
| 390 30.51282051 | |
| 391 30.69053708 | |
| 392 30.86734694 | |
| 393 30.78880407 | |
| 394 30.71065990 | |
| 395 30.63291139 | |
| 396 30.80808081 | |
| 397 30.73047859 | |
| 398 30.65326633 | |
| 399 30.82706767 | |
| 400 30.75000000 | |
| 401 30.92269327 | |
| 402 30.84577114 | |
| 403 30.76923077 | |
| 404 30.69306931 | |
| 405 30.61728395 | |
| 406 30.78817734 | |
| 407 30.71253071 | |
| 408 30.63725490 | |
| 409 30.80684597 | |
| 410 30.73170732 | |
| 411 30.65693431 | |
| 412 30.58252427 | |
| 413 30.75060533 | |
| 414 30.91787440 | |
| 415 31.08433735 | |
| 416 31.00961538 | |
| 417 30.93525180 | |
| 418 30.86124402 | |
| 419 31.02625298 | |
| 420 31.19047619 | |
| 421 31.11638955 | |
| 422 31.04265403 | |
| 423 30.96926714 | |
| 424 31.13207547 | |
| 425 31.29411765 | |
| 426 31.22065728 | |
| 427 31.14754098 | |
| 428 31.07476636 | |
| 429 31.23543124 | |
| 430 31.16279070 | |
| 431 31.32250580 | |
| 432 31.48148148 | |
| 433 31.40877598 | |
| 434 31.56682028 | |
| 435 31.49425287 | |
| 436 31.42201835 | |
| 437 31.35011442 | |
| 438 31.50684932 | |
| 439 31.66287016 | |
| 440 31.59090909 | |
| 441 31.51927438 | |
| 442 31.44796380 | |
| 443 31.37697517 | |
| 444 31.30630631 | |
| 445 31.23595506 | |
| 446 31.16591928 | |
| 447 31.09619687 | |
| 448 31.02678571 | |
| 449 30.95768374 | |
| 450 30.88888889 | |
| 451 30.82039911 | |
| 452 30.97345133 | |
| 453 31.12582781 | |
| 454 31.05726872 | |
| 455 30.98901099 | |
| 456 31.14035088 | |
| 457 31.07221007 | |
| 458 31.00436681 | |
| 459 31.15468410 | |
| 460 31.30434783 | |
| 461 31.23644252 | |
| 462 31.16883117 | |
| 463 31.31749460 | |
| 464 31.25000000 | |
| 465 31.18279570 | |
| 466 31.11587983 | |
| 467 31.26338330 | |
| 468 31.19658120 | |
| 469 31.34328358 | |
| 470 31.27659574 | |
| 471 31.21019108 | |
| 472 31.14406780 | |
| 473 31.28964059 | |
| 474 31.22362869 | |
| 475 31.15789474 | |
| 476 31.30252101 | |
| 477 31.23689727 | |
| 478 31.38075314 | |
| 479 31.52400835 | |
| 480 31.45833333 | |
| 481 31.39293139 | |
| 482 31.32780083 | |
| 483 31.26293996 | |
| 484 31.19834711 | |
| 485 31.13402062 | |
| 486 31.06995885 | |
| 487 31.21149897 | |
| 488 31.14754098 | |
| 489 31.08384458 | |
| 490 31.22448980 | |
| 491 31.36456212 | |
| 492 31.30081301 | |
| 493 31.23732252 | |
| 494 31.37651822 | |
| 495 31.51515152 | |
| 496 31.65322581 | |
| 497 31.58953722 | |
| 498 31.52610442 | |
| 499 31.46292585 | |
| 500 31.60000000 | |
| 501 31.53692615 | |
| 502 31.47410359 | |
| 503 31.61033797 | |
| 504 31.54761905 | |
| 505 31.48514851 | |
| 506 31.62055336 | |
| 507 31.55818540 | |
| 508 31.49606299 | |
| 509 31.43418468 | |
| 510 31.56862745 | |
| 511 31.50684932 | |
| 512 31.44531250 | |
| 513 31.38401559 | |
| 514 31.32295720 | |
| 515 31.26213592 | |
| 516 31.20155039 | |
| 517 31.33462282 | |
| 518 31.46718147 | |
| 519 31.40655106 | |
| 520 31.34615385 | |
| 521 31.28598848 | |
| 522 31.41762452 | |
| 523 31.54875717 | |
| 524 31.67938931 | |
| 525 31.61904762 | |
| 526 31.55893536 | |
| 527 31.68880455 | |
| 528 31.62878788 | |
| 529 31.56899811 | |
| 530 31.69811321 | |
| 531 31.63841808 | |
| 532 31.57894737 | |
| 533 31.51969981 | |
| 534 31.46067416 | |
| 535 31.40186916 | |
| 536 31.34328358 | |
| 537 31.47113594 | |
| 538 31.59851301 | |
| 539 31.53988868 | |
| 540 31.48148148 | |
| 541 31.60813309 | |
| 542 31.73431734 | |
| 543 31.86003683 | |
| 544 31.80147059 | |
| 545 31.74311927 | |
| 546 31.68498168 | |
| 547 31.62705667 | |
| 548 31.75182482 | |
| 549 31.69398907 | |
| 550 31.63636364 | |
| 551 31.57894737 | |
| 552 31.70289855 | |
| 553 31.64556962 | |
| 554 31.58844765 | |
| 555 31.53153153 | |
| 556 31.47482014 | |
| 557 31.41831239 | |
| 558 31.54121864 | |
| 559 31.48479428 | |
| 560 31.42857143 | |
| 561 31.37254902 | |
| 562 31.31672598 | |
| 563 31.43872114 | |
| 564 31.38297872 | |
| 565 31.32743363 | |
| 566 31.27208481 | |
| 567 31.21693122 | |
| 568 31.33802817 | |
| 569 31.28295255 | |
| 570 31.22807018 | |
| 571 31.34851138 | |
| 572 31.29370629 | |
| 573 31.23909250 | |
| 574 31.35888502 | |
| 575 31.30434783 | |
| 576 31.42361111 | |
| 577 31.54246101 | |
| 578 31.48788927 | |
| 579 31.43350604 | |
| 580 31.37931034 | |
| 581 31.32530120 | |
| 582 31.27147766 | |
| 583 31.21783877 | |
| 584 31.33561644 | |
| 585 31.28205128 | |
| 586 31.22866894 | |
| 587 31.17546848 | |
| 588 31.12244898 | |
| 589 31.06960951 | |
| 590 31.01694915 | |
| 591 30.96446701 | |
| 592 30.91216216 | |
| 593 30.86003373 | |
| 594 30.97643098 | |
| 595 30.92436975 | |
| 596 31.04026846 | |
| 597 30.98827471 | |
| 598 30.93645485 | |
| 599 31.05175292 | |
| 600 31.00000000 | |
| 601 31.11480865 | |
| 602 31.06312292 | |
| 603 31.17744610 | |
| 604 31.12582781 | |
| 605 31.23966942 | |
| 606 31.18811881 | |
| 607 31.30148270 | |
| 608 31.25000000 | |
| 609 31.19868637 | |
| 610 31.14754098 | |
| 611 31.09656301 | |
| 612 31.04575163 | |
| 613 30.99510604 | |
| 614 31.10749186 | |
| 615 31.05691057 | |
| 616 31.16883117 | |
| 617 31.28038898 | |
| 618 31.22977346 | |
| 619 31.17932149 | |
| 620 31.12903226 | |
| 621 31.07890499 | |
| 622 31.18971061 | |
| 623 31.13964687 | |
| 624 31.08974359 | |
| 625 31.04000000 | |
| 626 31.15015974 | |
| 627 31.10047847 | |
| 628 31.05095541 | |
| 629 31.16057234 | |
| 630 31.11111111 | |
| 631 31.06180666 | |
| 632 31.17088608 | |
| 633 31.12164297 | |
| 634 31.07255521 | |
| 635 31.02362205 | |
| 636 30.97484277 | |
| 637 30.92621664 | |
| 638 30.87774295 | |
| 639 30.82942097 | |
| 640 30.93750000 | |
| 641 30.88923557 | |
| 642 30.99688474 | |
| 643 30.94867807 | |
| 644 31.05590062 | |
| 645 31.00775194 | |
| 646 30.95975232 | |
| 647 30.91190108 | |
| 648 30.86419753 | |
| 649 30.81664099 | |
| 650 30.76923077 | |
| 651 30.72196621 | |
| 652 30.67484663 | |
| 653 30.62787136 | |
| 654 30.73394495 | |
| 655 30.68702290 | |
| 656 30.79268293 | |
| 657 30.74581431 | |
| 658 30.85106383 | |
| 659 30.80424886 | |
| 660 30.75757576 | |
| 661 30.71104387 | |
| 662 30.66465257 | |
| 663 30.61840121 | |
| 664 30.57228916 | |
| 665 30.52631579 | |
| 666 30.48048048 | |
| 667 30.58470765 | |
| 668 30.68862275 | |
| 669 30.64275037 | |
| 670 30.74626866 | |
| 671 30.70044709 | |
| 672 30.65476190 | |
| 673 30.60921248 | |
| 674 30.56379822 | |
| 675 30.51851852 | |
| 676 30.47337278 | |
| 677 30.42836041 | |
| 678 30.38348083 | |
| 679 30.33873343 | |
| 680 30.44117647 | |
| 681 30.54331865 | |
| 682 30.49853372 | |
| 683 30.60029283 | |
| 684 30.55555556 | |
| 685 30.51094891 | |
| 686 30.46647230 | |
| 687 30.42212518 | |
| 688 30.37790698 | |
| 689 30.33381713 | |
| 690 30.28985507 | |
| 691 30.39073806 | |
| 692 30.34682081 | |
| 693 30.30303030 | |
| 694 30.40345821 | |
| 695 30.35971223 | |
| 696 30.45977011 | |
| 697 30.41606887 | |
| 698 30.37249284 | |
| 699 30.47210300 | |
| 700 30.42857143 | |
| 701 30.52781740 | |
| 702 30.62678063 | |
| 703 30.72546230 | |
| 704 30.68181818 | |
| 705 30.63829787 | |
| 706 30.59490085 | |
| 707 30.55162659 | |
| 708 30.50847458 | |
| 709 30.46544429 | |
| 710 30.56338028 | |
| 711 30.66104079 | |
| 712 30.61797753 | |
| 713 30.71528752 | |
| 714 30.81232493 | |
| 715 30.76923077 | |
| 716 30.72625698 | |
| 717 30.68340307 | |
| 718 30.64066852 | |
| 719 30.59805285 | |
| 720 30.55555556 | |
| 721 30.51317614 | |
| 722 30.47091413 | |
| 723 30.56708160 | |
| 724 30.52486188 | |
| 725 30.62068966 | |
| 726 30.71625344 | |
| 727 30.67400275 | |
| 728 30.63186813 | |
| 729 30.72702332 | |
| 730 30.68493151 | |
| 731 30.64295486 | |
| 732 30.73770492 | |
| 733 30.83219645 | |
| 734 30.79019074 | |
| 735 30.74829932 | |
| 736 30.70652174 | |
| 737 30.80054274 | |
| 738 30.75880759 | |
| 739 30.85250338 | |
| 740 30.94594595 | |
| 741 30.90418354 | |
| 742 30.99730458 | |
| 743 31.09017497 | |
| 744 31.18279570 | |
| 745 31.14093960 | |
| 746 31.09919571 | |
| 747 31.05756359 | |
| 748 31.01604278 | |
| 749 31.10814419 | |
| 750 31.20000000 | |
| Final result: 31.2000 ±1.6929 | |
| Random chance: 25.0000 ±1.5822 | |