| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 1548 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 1548 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 50.00000000 | |
| 5 40.00000000 | |
| 6 33.33333333 | |
| 7 42.85714286 | |
| 8 37.50000000 | |
| 9 33.33333333 | |
| 10 30.00000000 | |
| 11 27.27272727 | |
| 12 25.00000000 | |
| 13 30.76923077 | |
| 14 28.57142857 | |
| 15 33.33333333 | |
| 16 37.50000000 | |
| 17 35.29411765 | |
| 18 33.33333333 | |
| 19 36.84210526 | |
| 20 35.00000000 | |
| 21 33.33333333 | |
| 22 31.81818182 | |
| 23 30.43478261 | |
| 24 29.16666667 | |
| 25 32.00000000 | |
| 26 30.76923077 | |
| 27 29.62962963 | |
| 28 32.14285714 | |
| 29 34.48275862 | |
| 30 36.66666667 | |
| 31 38.70967742 | |
| 32 40.62500000 | |
| 33 39.39393939 | |
| 34 38.23529412 | |
| 35 40.00000000 | |
| 36 38.88888889 | |
| 37 37.83783784 | |
| 38 36.84210526 | |
| 39 35.89743590 | |
| 40 35.00000000 | |
| 41 34.14634146 | |
| 42 33.33333333 | |
| 43 32.55813953 | |
| 44 31.81818182 | |
| 45 31.11111111 | |
| 46 30.43478261 | |
| 47 29.78723404 | |
| 48 31.25000000 | |
| 49 32.65306122 | |
| 50 34.00000000 | |
| 51 33.33333333 | |
| 52 32.69230769 | |
| 53 32.07547170 | |
| 54 31.48148148 | |
| 55 32.72727273 | |
| 56 33.92857143 | |
| 57 35.08771930 | |
| 58 34.48275862 | |
| 59 33.89830508 | |
| 60 33.33333333 | |
| 61 32.78688525 | |
| 62 32.25806452 | |
| 63 31.74603175 | |
| 64 31.25000000 | |
| 65 32.30769231 | |
| 66 33.33333333 | |
| 67 32.83582090 | |
| 68 32.35294118 | |
| 69 31.88405797 | |
| 70 32.85714286 | |
| 71 32.39436620 | |
| 72 33.33333333 | |
| 73 32.87671233 | |
| 74 33.78378378 | |
| 75 34.66666667 | |
| 76 35.52631579 | |
| 77 36.36363636 | |
| 78 37.17948718 | |
| 79 36.70886076 | |
| 80 37.50000000 | |
| 81 38.27160494 | |
| 82 37.80487805 | |
| 83 38.55421687 | |
| 84 39.28571429 | |
| 85 38.82352941 | |
| 86 38.37209302 | |
| 87 39.08045977 | |
| 88 39.77272727 | |
| 89 39.32584270 | |
| 90 38.88888889 | |
| 91 38.46153846 | |
| 92 38.04347826 | |
| 93 37.63440860 | |
| 94 38.29787234 | |
| 95 37.89473684 | |
| 96 38.54166667 | |
| 97 38.14432990 | |
| 98 37.75510204 | |
| 99 37.37373737 | |
| 100 38.00000000 | |
| 101 37.62376238 | |
| 102 37.25490196 | |
| 103 36.89320388 | |
| 104 36.53846154 | |
| 105 37.14285714 | |
| 106 36.79245283 | |
| 107 36.44859813 | |
| 108 36.11111111 | |
| 109 35.77981651 | |
| 110 35.45454545 | |
| 111 35.13513514 | |
| 112 34.82142857 | |
| 113 34.51327434 | |
| 114 35.08771930 | |
| 115 35.65217391 | |
| 116 35.34482759 | |
| 117 35.04273504 | |
| 118 34.74576271 | |
| 119 34.45378151 | |
| 120 34.16666667 | |
| 121 33.88429752 | |
| 122 33.60655738 | |
| 123 33.33333333 | |
| 124 33.06451613 | |
| 125 33.60000000 | |
| 126 33.33333333 | |
| 127 33.07086614 | |
| 128 32.81250000 | |
| 129 32.55813953 | |
| 130 32.30769231 | |
| 131 32.82442748 | |
| 132 33.33333333 | |
| 133 33.83458647 | |
| 134 33.58208955 | |
| 135 33.33333333 | |
| 136 33.08823529 | |
| 137 32.84671533 | |
| 138 32.60869565 | |
| 139 32.37410072 | |
| 140 32.14285714 | |
| 141 32.62411348 | |
| 142 32.39436620 | |
| 143 32.86713287 | |
| 144 33.33333333 | |
| 145 33.10344828 | |
| 146 32.87671233 | |
| 147 32.65306122 | |
| 148 33.10810811 | |
| 149 32.88590604 | |
| 150 32.66666667 | |
| 151 32.45033113 | |
| 152 32.23684211 | |
| 153 32.02614379 | |
| 154 31.81818182 | |
| 155 31.61290323 | |
| 156 31.41025641 | |
| 157 31.84713376 | |
| 158 31.64556962 | |
| 159 31.44654088 | |
| 160 31.87500000 | |
| 161 31.67701863 | |
| 162 31.48148148 | |
| 163 31.28834356 | |
| 164 31.70731707 | |
| 165 32.12121212 | |
| 166 31.92771084 | |
| 167 32.33532934 | |
| 168 32.14285714 | |
| 169 31.95266272 | |
| 170 31.76470588 | |
| 171 31.57894737 | |
| 172 31.39534884 | |
| 173 31.21387283 | |
| 174 31.03448276 | |
| 175 30.85714286 | |
| 176 30.68181818 | |
| 177 30.50847458 | |
| 178 30.33707865 | |
| 179 30.72625698 | |
| 180 30.55555556 | |
| 181 30.38674033 | |
| 182 30.21978022 | |
| 183 30.05464481 | |
| 184 29.89130435 | |
| 185 30.27027027 | |
| 186 30.64516129 | |
| 187 30.48128342 | |
| 188 30.31914894 | |
| 189 30.15873016 | |
| 190 30.00000000 | |
| 191 30.36649215 | |
| 192 30.20833333 | |
| 193 30.05181347 | |
| 194 30.41237113 | |
| 195 30.76923077 | |
| 196 30.61224490 | |
| 197 30.45685279 | |
| 198 30.30303030 | |
| 199 30.15075377 | |
| 200 30.00000000 | |
| 201 30.34825871 | |
| 202 30.19801980 | |
| 203 30.04926108 | |
| 204 29.90196078 | |
| 205 30.24390244 | |
| 206 30.09708738 | |
| 207 29.95169082 | |
| 208 29.80769231 | |
| 209 29.66507177 | |
| 210 29.52380952 | |
| 211 29.85781991 | |
| 212 29.71698113 | |
| 213 29.57746479 | |
| 214 29.43925234 | |
| 215 29.30232558 | |
| 216 29.62962963 | |
| 217 29.49308756 | |
| 218 29.35779817 | |
| 219 29.22374429 | |
| 220 29.09090909 | |
| 221 28.95927602 | |
| 222 29.27927928 | |
| 223 29.14798206 | |
| 224 29.01785714 | |
| 225 29.33333333 | |
| 226 29.20353982 | |
| 227 29.51541850 | |
| 228 29.82456140 | |
| 229 29.69432314 | |
| 230 29.56521739 | |
| 231 29.43722944 | |
| 232 29.74137931 | |
| 233 29.61373391 | |
| 234 29.91452991 | |
| 235 29.78723404 | |
| 236 29.66101695 | |
| 237 29.53586498 | |
| 238 29.41176471 | |
| 239 29.28870293 | |
| 240 29.58333333 | |
| 241 29.46058091 | |
| 242 29.75206612 | |
| 243 30.04115226 | |
| 244 29.91803279 | |
| 245 30.20408163 | |
| 246 30.08130081 | |
| 247 29.95951417 | |
| 248 30.24193548 | |
| 249 30.12048193 | |
| 250 30.00000000 | |
| 251 29.88047809 | |
| 252 29.76190476 | |
| 253 29.64426877 | |
| 254 29.52755906 | |
| 255 29.80392157 | |
| 256 29.68750000 | |
| 257 29.96108949 | |
| 258 29.84496124 | |
| 259 29.72972973 | |
| 260 29.61538462 | |
| 261 29.50191571 | |
| 262 29.38931298 | |
| 263 29.27756654 | |
| 264 29.16666667 | |
| 265 29.05660377 | |
| 266 29.32330827 | |
| 267 29.21348315 | |
| 268 29.10447761 | |
| 269 28.99628253 | |
| 270 28.88888889 | |
| 271 28.78228782 | |
| 272 29.04411765 | |
| 273 28.93772894 | |
| 274 28.83211679 | |
| 275 28.72727273 | |
| 276 28.98550725 | |
| 277 29.24187726 | |
| 278 29.49640288 | |
| 279 29.39068100 | |
| 280 29.64285714 | |
| 281 29.53736655 | |
| 282 29.43262411 | |
| 283 29.32862191 | |
| 284 29.22535211 | |
| 285 29.12280702 | |
| 286 29.02097902 | |
| 287 29.26829268 | |
| 288 29.16666667 | |
| 289 29.06574394 | |
| 290 29.31034483 | |
| 291 29.55326460 | |
| 292 29.79452055 | |
| 293 30.03412969 | |
| 294 30.27210884 | |
| 295 30.50847458 | |
| 296 30.40540541 | |
| 297 30.30303030 | |
| 298 30.53691275 | |
| 299 30.76923077 | |
| 300 31.00000000 | |
| 301 31.22923588 | |
| 302 31.45695364 | |
| 303 31.35313531 | |
| 304 31.57894737 | |
| 305 31.80327869 | |
| 306 31.69934641 | |
| 307 31.59609121 | |
| 308 31.49350649 | |
| 309 31.39158576 | |
| 310 31.29032258 | |
| 311 31.51125402 | |
| 312 31.73076923 | |
| 313 31.94888179 | |
| 314 32.16560510 | |
| 315 32.06349206 | |
| 316 31.96202532 | |
| 317 31.86119874 | |
| 318 31.76100629 | |
| 319 31.97492163 | |
| 320 31.87500000 | |
| 321 31.77570093 | |
| 322 31.67701863 | |
| 323 31.57894737 | |
| 324 31.48148148 | |
| 325 31.69230769 | |
| 326 31.59509202 | |
| 327 31.80428135 | |
| 328 32.01219512 | |
| 329 31.91489362 | |
| 330 31.81818182 | |
| 331 31.72205438 | |
| 332 31.92771084 | |
| 333 31.83183183 | |
| 334 31.73652695 | |
| 335 31.94029851 | |
| 336 31.84523810 | |
| 337 31.75074184 | |
| 338 31.65680473 | |
| 339 31.56342183 | |
| 340 31.76470588 | |
| 341 31.67155425 | |
| 342 31.57894737 | |
| 343 31.77842566 | |
| 344 31.68604651 | |
| 345 31.59420290 | |
| 346 31.79190751 | |
| 347 31.98847262 | |
| 348 31.89655172 | |
| 349 31.80515759 | |
| 350 32.00000000 | |
| 351 31.90883191 | |
| 352 31.81818182 | |
| 353 32.01133144 | |
| 354 31.92090395 | |
| 355 31.83098592 | |
| 356 32.02247191 | |
| 357 31.93277311 | |
| 358 31.84357542 | |
| 359 31.75487465 | |
| 360 31.94444444 | |
| 361 32.13296399 | |
| 362 32.32044199 | |
| 363 32.50688705 | |
| 364 32.41758242 | |
| 365 32.32876712 | |
| 366 32.24043716 | |
| 367 32.15258856 | |
| 368 32.06521739 | |
| 369 31.97831978 | |
| 370 32.16216216 | |
| 371 32.07547170 | |
| 372 31.98924731 | |
| 373 31.90348525 | |
| 374 32.08556150 | |
| 375 32.26666667 | |
| 376 32.18085106 | |
| 377 32.09549072 | |
| 378 32.27513228 | |
| 379 32.18997361 | |
| 380 32.10526316 | |
| 381 32.28346457 | |
| 382 32.46073298 | |
| 383 32.63707572 | |
| 384 32.55208333 | |
| 385 32.72727273 | |
| 386 32.64248705 | |
| 387 32.55813953 | |
| 388 32.47422680 | |
| 389 32.64781491 | |
| 390 32.56410256 | |
| 391 32.73657289 | |
| 392 32.90816327 | |
| 393 32.82442748 | |
| 394 32.74111675 | |
| 395 32.65822785 | |
| 396 32.82828283 | |
| 397 32.74559194 | |
| 398 32.66331658 | |
| 399 32.83208020 | |
| 400 32.75000000 | |
| 401 32.91770574 | |
| 402 33.08457711 | |
| 403 33.00248139 | |
| 404 32.92079208 | |
| 405 32.83950617 | |
| 406 32.75862069 | |
| 407 32.67813268 | |
| 408 32.59803922 | |
| 409 32.76283619 | |
| 410 32.68292683 | |
| 411 32.60340633 | |
| 412 32.52427184 | |
| 413 32.68765133 | |
| 414 32.85024155 | |
| 415 33.01204819 | |
| 416 33.17307692 | |
| 417 33.09352518 | |
| 418 33.01435407 | |
| 419 33.17422434 | |
| 420 33.33333333 | |
| 421 33.25415677 | |
| 422 33.17535545 | |
| 423 33.09692671 | |
| 424 33.25471698 | |
| 425 33.41176471 | |
| 426 33.56807512 | |
| 427 33.48946136 | |
| 428 33.41121495 | |
| 429 33.56643357 | |
| 430 33.48837209 | |
| 431 33.64269142 | |
| 432 33.79629630 | |
| 433 33.71824480 | |
| 434 33.87096774 | |
| 435 33.79310345 | |
| 436 33.71559633 | |
| 437 33.63844394 | |
| 438 33.78995434 | |
| 439 33.94077449 | |
| 440 33.86363636 | |
| 441 33.78684807 | |
| 442 33.71040724 | |
| 443 33.63431151 | |
| 444 33.55855856 | |
| 445 33.48314607 | |
| 446 33.40807175 | |
| 447 33.33333333 | |
| 448 33.25892857 | |
| 449 33.18485523 | |
| 450 33.11111111 | |
| 451 33.03769401 | |
| 452 33.18584071 | |
| 453 33.33333333 | |
| 454 33.48017621 | |
| 455 33.40659341 | |
| 456 33.55263158 | |
| 457 33.69803063 | |
| 458 33.62445415 | |
| 459 33.76906318 | |
| 460 33.91304348 | |
| 461 33.83947939 | |
| 462 33.76623377 | |
| 463 33.90928726 | |
| 464 34.05172414 | |
| 465 33.97849462 | |
| 466 33.90557940 | |
| 467 34.04710921 | |
| 468 33.97435897 | |
| 469 34.11513859 | |
| 470 34.25531915 | |
| 471 34.18259023 | |
| 472 34.11016949 | |
| 473 34.24947146 | |
| 474 34.17721519 | |
| 475 34.10526316 | |
| 476 34.24369748 | |
| 477 34.17190776 | |
| 478 34.30962343 | |
| 479 34.44676409 | |
| 480 34.37500000 | |
| 481 34.51143451 | |
| 482 34.43983402 | |
| 483 34.36853002 | |
| 484 34.29752066 | |
| 485 34.22680412 | |
| 486 34.15637860 | |
| 487 34.29158111 | |
| 488 34.22131148 | |
| 489 34.15132924 | |
| 490 34.28571429 | |
| 491 34.41955193 | |
| 492 34.34959350 | |
| 493 34.27991886 | |
| 494 34.41295547 | |
| 495 34.54545455 | |
| 496 34.67741935 | |
| 497 34.60764588 | |
| 498 34.53815261 | |
| 499 34.46893788 | |
| 500 34.60000000 | |
| 501 34.53093812 | |
| 502 34.66135458 | |
| 503 34.59244533 | |
| 504 34.72222222 | |
| 505 34.65346535 | |
| 506 34.78260870 | |
| 507 34.71400394 | |
| 508 34.64566929 | |
| 509 34.57760314 | |
| 510 34.70588235 | |
| 511 34.63796477 | |
| 512 34.57031250 | |
| 513 34.50292398 | |
| 514 34.43579767 | |
| 515 34.36893204 | |
| 516 34.30232558 | |
| 517 34.42940039 | |
| 518 34.55598456 | |
| 519 34.48940270 | |
| 520 34.42307692 | |
| 521 34.35700576 | |
| 522 34.29118774 | |
| 523 34.41682600 | |
| 524 34.54198473 | |
| 525 34.47619048 | |
| 526 34.41064639 | |
| 527 34.53510436 | |
| 528 34.46969697 | |
| 529 34.59357278 | |
| 530 34.71698113 | |
| 531 34.65160075 | |
| 532 34.58646617 | |
| 533 34.70919325 | |
| 534 34.64419476 | |
| 535 34.76635514 | |
| 536 34.88805970 | |
| 537 35.00931099 | |
| 538 35.13011152 | |
| 539 35.06493506 | |
| 540 35.00000000 | |
| 541 34.93530499 | |
| 542 35.05535055 | |
| 543 34.99079190 | |
| 544 34.92647059 | |
| 545 34.86238532 | |
| 546 34.79853480 | |
| 547 34.73491773 | |
| 548 34.85401460 | |
| 549 34.79052823 | |
| 550 34.72727273 | |
| 551 34.84573503 | |
| 552 34.78260870 | |
| 553 34.71971067 | |
| 554 34.65703971 | |
| 555 34.59459459 | |
| 556 34.53237410 | |
| 557 34.47037702 | |
| 558 34.58781362 | |
| 559 34.52593918 | |
| 560 34.46428571 | |
| 561 34.40285205 | |
| 562 34.34163701 | |
| 563 34.45825933 | |
| 564 34.39716312 | |
| 565 34.51327434 | |
| 566 34.45229682 | |
| 567 34.39153439 | |
| 568 34.50704225 | |
| 569 34.44639719 | |
| 570 34.38596491 | |
| 571 34.50087566 | |
| 572 34.44055944 | |
| 573 34.38045375 | |
| 574 34.49477352 | |
| 575 34.43478261 | |
| 576 34.54861111 | |
| 577 34.66204506 | |
| 578 34.60207612 | |
| 579 34.54231434 | |
| 580 34.48275862 | |
| 581 34.42340792 | |
| 582 34.36426117 | |
| 583 34.47684391 | |
| 584 34.58904110 | |
| 585 34.52991453 | |
| 586 34.47098976 | |
| 587 34.41226576 | |
| 588 34.35374150 | |
| 589 34.29541596 | |
| 590 34.23728814 | |
| 591 34.17935702 | |
| 592 34.12162162 | |
| 593 34.06408094 | |
| 594 34.17508418 | |
| 595 34.11764706 | |
| 596 34.22818792 | |
| 597 34.17085427 | |
| 598 34.11371237 | |
| 599 34.22370618 | |
| 600 34.16666667 | |
| 601 34.27620632 | |
| 602 34.21926910 | |
| 603 34.32835821 | |
| 604 34.27152318 | |
| 605 34.21487603 | |
| 606 34.15841584 | |
| 607 34.26688633 | |
| 608 34.21052632 | |
| 609 34.15435140 | |
| 610 34.09836066 | |
| 611 34.04255319 | |
| 612 33.98692810 | |
| 613 34.09461664 | |
| 614 34.20195440 | |
| 615 34.14634146 | |
| 616 34.25324675 | |
| 617 34.35980551 | |
| 618 34.30420712 | |
| 619 34.24878837 | |
| 620 34.19354839 | |
| 621 34.13848631 | |
| 622 34.08360129 | |
| 623 34.02889246 | |
| 624 33.97435897 | |
| 625 33.92000000 | |
| 626 34.02555911 | |
| 627 33.97129187 | |
| 628 33.91719745 | |
| 629 34.02225755 | |
| 630 34.12698413 | |
| 631 34.07290016 | |
| 632 34.17721519 | |
| 633 34.12322275 | |
| 634 34.06940063 | |
| 635 34.01574803 | |
| 636 33.96226415 | |
| 637 33.90894819 | |
| 638 33.85579937 | |
| 639 33.80281690 | |
| 640 33.90625000 | |
| 641 33.85335413 | |
| 642 33.95638629 | |
| 643 33.90357698 | |
| 644 34.00621118 | |
| 645 33.95348837 | |
| 646 33.90092879 | |
| 647 33.84853168 | |
| 648 33.79629630 | |
| 649 33.74422188 | |
| 650 33.69230769 | |
| 651 33.64055300 | |
| 652 33.74233129 | |
| 653 33.69065850 | |
| 654 33.79204893 | |
| 655 33.74045802 | |
| 656 33.84146341 | |
| 657 33.78995434 | |
| 658 33.89057751 | |
| 659 33.83915023 | |
| 660 33.78787879 | |
| 661 33.73676248 | |
| 662 33.68580060 | |
| 663 33.63499246 | |
| 664 33.73493976 | |
| 665 33.68421053 | |
| 666 33.63363363 | |
| 667 33.58320840 | |
| 668 33.68263473 | |
| 669 33.63228700 | |
| 670 33.73134328 | |
| 671 33.68107303 | |
| 672 33.63095238 | |
| 673 33.58098068 | |
| 674 33.53115727 | |
| 675 33.48148148 | |
| 676 33.43195266 | |
| 677 33.38257016 | |
| 678 33.33333333 | |
| 679 33.28424153 | |
| 680 33.38235294 | |
| 681 33.33333333 | |
| 682 33.28445748 | |
| 683 33.38213763 | |
| 684 33.33333333 | |
| 685 33.28467153 | |
| 686 33.23615160 | |
| 687 33.33333333 | |
| 688 33.28488372 | |
| 689 33.23657475 | |
| 690 33.18840580 | |
| 691 33.14037627 | |
| 692 33.09248555 | |
| 693 33.04473304 | |
| 694 32.99711816 | |
| 695 32.94964029 | |
| 696 33.04597701 | |
| 697 32.99856528 | |
| 698 32.95128940 | |
| 699 33.04721030 | |
| 700 33.00000000 | |
| 701 33.09557775 | |
| 702 33.19088319 | |
| 703 33.28591750 | |
| 704 33.23863636 | |
| 705 33.19148936 | |
| 706 33.14447592 | |
| 707 33.09759547 | |
| 708 33.05084746 | |
| 709 33.00423131 | |
| 710 33.09859155 | |
| 711 33.19268636 | |
| 712 33.14606742 | |
| 713 33.23983170 | |
| 714 33.33333333 | |
| 715 33.28671329 | |
| 716 33.24022346 | |
| 717 33.19386332 | |
| 718 33.14763231 | |
| 719 33.10152990 | |
| 720 33.05555556 | |
| 721 33.00970874 | |
| 722 32.96398892 | |
| 723 33.05670816 | |
| 724 33.01104972 | |
| 725 33.10344828 | |
| 726 33.19559229 | |
| 727 33.14993122 | |
| 728 33.10439560 | |
| 729 33.05898491 | |
| 730 33.01369863 | |
| 731 32.96853625 | |
| 732 33.06010929 | |
| 733 33.15143247 | |
| 734 33.24250681 | |
| 735 33.19727891 | |
| 736 33.15217391 | |
| 737 33.24287653 | |
| 738 33.19783198 | |
| 739 33.28822733 | |
| 740 33.37837838 | |
| 741 33.33333333 | |
| 742 33.42318059 | |
| 743 33.51278600 | |
| 744 33.60215054 | |
| 745 33.55704698 | |
| 746 33.51206434 | |
| 747 33.46720214 | |
| 748 33.42245989 | |
| 749 33.37783712 | |
| 750 33.46666667 | |
| Final result: 33.4667 ±1.7242 | |
| Random chance: 25.0000 ±1.5822 | |