| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 817 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 817 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 25.00000000 | |
| 5 40.00000000 | |
| 6 33.33333333 | |
| 7 28.57142857 | |
| 8 37.50000000 | |
| 9 33.33333333 | |
| 10 30.00000000 | |
| 11 27.27272727 | |
| 12 33.33333333 | |
| 13 38.46153846 | |
| 14 35.71428571 | |
| 15 40.00000000 | |
| 16 37.50000000 | |
| 17 35.29411765 | |
| 18 33.33333333 | |
| 19 31.57894737 | |
| 20 30.00000000 | |
| 21 28.57142857 | |
| 22 27.27272727 | |
| 23 26.08695652 | |
| 24 29.16666667 | |
| 25 28.00000000 | |
| 26 26.92307692 | |
| 27 25.92592593 | |
| 28 25.00000000 | |
| 29 27.58620690 | |
| 30 26.66666667 | |
| 31 25.80645161 | |
| 32 28.12500000 | |
| 33 30.30303030 | |
| 34 29.41176471 | |
| 35 28.57142857 | |
| 36 27.77777778 | |
| 37 29.72972973 | |
| 38 28.94736842 | |
| 39 30.76923077 | |
| 40 30.00000000 | |
| 41 29.26829268 | |
| 42 28.57142857 | |
| 43 27.90697674 | |
| 44 29.54545455 | |
| 45 31.11111111 | |
| 46 30.43478261 | |
| 47 31.91489362 | |
| 48 33.33333333 | |
| 49 32.65306122 | |
| 50 34.00000000 | |
| 51 33.33333333 | |
| 52 32.69230769 | |
| 53 32.07547170 | |
| 54 31.48148148 | |
| 55 32.72727273 | |
| 56 32.14285714 | |
| 57 31.57894737 | |
| 58 32.75862069 | |
| 59 33.89830508 | |
| 60 35.00000000 | |
| 61 34.42622951 | |
| 62 35.48387097 | |
| 63 34.92063492 | |
| 64 34.37500000 | |
| 65 33.84615385 | |
| 66 33.33333333 | |
| 67 32.83582090 | |
| 68 32.35294118 | |
| 69 33.33333333 | |
| 70 32.85714286 | |
| 71 33.80281690 | |
| 72 34.72222222 | |
| 73 34.24657534 | |
| 74 33.78378378 | |
| 75 33.33333333 | |
| 76 32.89473684 | |
| 77 32.46753247 | |
| 78 32.05128205 | |
| 79 31.64556962 | |
| 80 32.50000000 | |
| 81 32.09876543 | |
| 82 32.92682927 | |
| 83 32.53012048 | |
| 84 32.14285714 | |
| 85 31.76470588 | |
| 86 31.39534884 | |
| 87 31.03448276 | |
| 88 30.68181818 | |
| 89 31.46067416 | |
| 90 31.11111111 | |
| 91 31.86813187 | |
| 92 31.52173913 | |
| 93 31.18279570 | |
| 94 30.85106383 | |
| 95 30.52631579 | |
| 96 31.25000000 | |
| 97 30.92783505 | |
| 98 30.61224490 | |
| 99 31.31313131 | |
| 100 31.00000000 | |
| 101 31.68316832 | |
| 102 31.37254902 | |
| 103 31.06796117 | |
| 104 30.76923077 | |
| 105 30.47619048 | |
| 106 31.13207547 | |
| 107 31.77570093 | |
| 108 31.48148148 | |
| 109 31.19266055 | |
| 110 31.81818182 | |
| 111 31.53153153 | |
| 112 32.14285714 | |
| 113 32.74336283 | |
| 114 33.33333333 | |
| 115 33.04347826 | |
| 116 32.75862069 | |
| 117 33.33333333 | |
| 118 33.05084746 | |
| 119 32.77310924 | |
| 120 33.33333333 | |
| 121 33.88429752 | |
| 122 33.60655738 | |
| 123 34.14634146 | |
| 124 34.67741935 | |
| 125 35.20000000 | |
| 126 35.71428571 | |
| 127 35.43307087 | |
| 128 35.15625000 | |
| 129 35.65891473 | |
| 130 36.15384615 | |
| 131 35.87786260 | |
| 132 35.60606061 | |
| 133 35.33834586 | |
| 134 35.07462687 | |
| 135 35.55555556 | |
| 136 35.29411765 | |
| 137 35.76642336 | |
| 138 36.23188406 | |
| 139 36.69064748 | |
| 140 36.42857143 | |
| 141 36.17021277 | |
| 142 35.91549296 | |
| 143 35.66433566 | |
| 144 36.11111111 | |
| 145 35.86206897 | |
| 146 35.61643836 | |
| 147 35.37414966 | |
| 148 35.13513514 | |
| 149 34.89932886 | |
| 150 34.66666667 | |
| 151 35.09933775 | |
| 152 35.52631579 | |
| 153 35.29411765 | |
| 154 35.06493506 | |
| 155 34.83870968 | |
| 156 35.25641026 | |
| 157 35.03184713 | |
| 158 34.81012658 | |
| 159 34.59119497 | |
| 160 34.37500000 | |
| 161 34.78260870 | |
| 162 35.18518519 | |
| 163 34.96932515 | |
| 164 34.75609756 | |
| 165 35.15151515 | |
| 166 35.54216867 | |
| 167 35.32934132 | |
| 168 35.71428571 | |
| 169 35.50295858 | |
| 170 35.29411765 | |
| 171 35.08771930 | |
| 172 34.88372093 | |
| 173 34.68208092 | |
| 174 35.05747126 | |
| 175 34.85714286 | |
| 176 35.22727273 | |
| 177 35.02824859 | |
| 178 34.83146067 | |
| 179 35.19553073 | |
| 180 35.00000000 | |
| 181 34.80662983 | |
| 182 34.61538462 | |
| 183 34.42622951 | |
| 184 34.23913043 | |
| 185 34.05405405 | |
| 186 34.40860215 | |
| 187 34.22459893 | |
| 188 34.04255319 | |
| 189 33.86243386 | |
| 190 33.68421053 | |
| 191 33.50785340 | |
| 192 33.33333333 | |
| 193 33.16062176 | |
| 194 33.50515464 | |
| 195 33.33333333 | |
| 196 33.16326531 | |
| 197 33.50253807 | |
| 198 33.83838384 | |
| 199 33.66834171 | |
| 200 33.50000000 | |
| 201 33.33333333 | |
| 202 33.66336634 | |
| 203 33.49753695 | |
| 204 33.33333333 | |
| 205 33.65853659 | |
| 206 33.49514563 | |
| 207 33.81642512 | |
| 208 33.65384615 | |
| 209 33.49282297 | |
| 210 33.33333333 | |
| 211 33.17535545 | |
| 212 33.01886792 | |
| 213 32.86384977 | |
| 214 32.71028037 | |
| 215 32.55813953 | |
| 216 32.87037037 | |
| 217 32.71889401 | |
| 218 32.56880734 | |
| 219 32.42009132 | |
| 220 32.27272727 | |
| 221 32.12669683 | |
| 222 31.98198198 | |
| 223 31.83856502 | |
| 224 32.14285714 | |
| 225 32.00000000 | |
| 226 31.85840708 | |
| 227 32.15859031 | |
| 228 32.01754386 | |
| 229 31.87772926 | |
| 230 31.73913043 | |
| 231 31.60173160 | |
| 232 31.89655172 | |
| 233 31.75965665 | |
| 234 32.05128205 | |
| 235 32.34042553 | |
| 236 32.62711864 | |
| 237 32.48945148 | |
| 238 32.77310924 | |
| 239 32.63598326 | |
| 240 32.50000000 | |
| 241 32.36514523 | |
| 242 32.23140496 | |
| 243 32.09876543 | |
| 244 31.96721311 | |
| 245 31.83673469 | |
| 246 31.70731707 | |
| 247 31.98380567 | |
| 248 32.25806452 | |
| 249 32.12851406 | |
| 250 32.00000000 | |
| 251 32.27091633 | |
| 252 32.53968254 | |
| 253 32.41106719 | |
| 254 32.28346457 | |
| 255 32.54901961 | |
| 256 32.42187500 | |
| 257 32.29571984 | |
| 258 32.17054264 | |
| 259 32.04633205 | |
| 260 31.92307692 | |
| 261 32.18390805 | |
| 262 32.06106870 | |
| 263 31.93916350 | |
| 264 32.19696970 | |
| 265 32.45283019 | |
| 266 32.33082707 | |
| 267 32.20973783 | |
| 268 32.08955224 | |
| 269 31.97026022 | |
| 270 31.85185185 | |
| 271 31.73431734 | |
| 272 31.61764706 | |
| 273 31.50183150 | |
| 274 31.38686131 | |
| 275 31.27272727 | |
| 276 31.15942029 | |
| 277 31.04693141 | |
| 278 31.29496403 | |
| 279 31.18279570 | |
| 280 31.42857143 | |
| 281 31.67259786 | |
| 282 31.56028369 | |
| 283 31.44876325 | |
| 284 31.33802817 | |
| 285 31.22807018 | |
| 286 31.46853147 | |
| 287 31.70731707 | |
| 288 31.94444444 | |
| 289 31.83391003 | |
| 290 31.72413793 | |
| 291 31.95876289 | |
| 292 31.84931507 | |
| 293 32.08191126 | |
| 294 31.97278912 | |
| 295 32.20338983 | |
| 296 32.43243243 | |
| 297 32.65993266 | |
| 298 32.55033557 | |
| 299 32.44147157 | |
| 300 32.33333333 | |
| 301 32.22591362 | |
| 302 32.11920530 | |
| 303 32.01320132 | |
| 304 31.90789474 | |
| 305 31.80327869 | |
| 306 31.69934641 | |
| 307 31.59609121 | |
| 308 31.81818182 | |
| 309 31.71521036 | |
| 310 31.61290323 | |
| 311 31.51125402 | |
| 312 31.73076923 | |
| 313 31.62939297 | |
| 314 31.52866242 | |
| 315 31.74603175 | |
| 316 31.64556962 | |
| 317 31.54574132 | |
| 318 31.44654088 | |
| 319 31.34796238 | |
| 320 31.25000000 | |
| 321 31.15264798 | |
| 322 31.36645963 | |
| 323 31.26934985 | |
| 324 31.17283951 | |
| 325 31.07692308 | |
| 326 30.98159509 | |
| 327 31.19266055 | |
| 328 31.40243902 | |
| 329 31.30699088 | |
| 330 31.51515152 | |
| 331 31.41993958 | |
| 332 31.62650602 | |
| 333 31.83183183 | |
| 334 31.73652695 | |
| 335 31.94029851 | |
| 336 32.14285714 | |
| 337 32.04747774 | |
| 338 31.95266272 | |
| 339 32.15339233 | |
| 340 32.05882353 | |
| 341 31.96480938 | |
| 342 32.16374269 | |
| 343 32.06997085 | |
| 344 32.26744186 | |
| 345 32.46376812 | |
| 346 32.36994220 | |
| 347 32.27665706 | |
| 348 32.18390805 | |
| 349 32.09169054 | |
| 350 32.00000000 | |
| 351 31.90883191 | |
| 352 31.81818182 | |
| 353 31.72804533 | |
| 354 31.63841808 | |
| 355 31.83098592 | |
| 356 31.74157303 | |
| 357 31.65266106 | |
| 358 31.84357542 | |
| 359 31.75487465 | |
| 360 31.66666667 | |
| 361 31.85595568 | |
| 362 31.76795580 | |
| 363 31.68044077 | |
| 364 31.86813187 | |
| 365 31.78082192 | |
| 366 31.69398907 | |
| 367 31.88010899 | |
| 368 32.06521739 | |
| 369 31.97831978 | |
| 370 31.89189189 | |
| 371 31.80592992 | |
| 372 31.98924731 | |
| 373 31.90348525 | |
| 374 32.08556150 | |
| 375 32.00000000 | |
| 376 31.91489362 | |
| 377 32.09549072 | |
| 378 32.01058201 | |
| 379 31.92612137 | |
| 380 32.10526316 | |
| 381 32.02099738 | |
| 382 32.19895288 | |
| 383 32.11488251 | |
| 384 32.03125000 | |
| 385 31.94805195 | |
| 386 31.86528497 | |
| 387 31.78294574 | |
| 388 31.95876289 | |
| 389 31.87660668 | |
| 390 31.79487179 | |
| 391 31.96930946 | |
| 392 31.88775510 | |
| 393 31.80661578 | |
| 394 31.97969543 | |
| 395 31.89873418 | |
| 396 32.07070707 | |
| 397 32.24181360 | |
| 398 32.16080402 | |
| 399 32.08020050 | |
| 400 32.00000000 | |
| 401 31.92019950 | |
| 402 32.08955224 | |
| 403 32.00992556 | |
| 404 31.93069307 | |
| 405 31.85185185 | |
| 406 31.77339901 | |
| 407 31.69533170 | |
| 408 31.61764706 | |
| 409 31.54034230 | |
| 410 31.46341463 | |
| 411 31.38686131 | |
| 412 31.55339806 | |
| 413 31.47699758 | |
| 414 31.64251208 | |
| 415 31.80722892 | |
| 416 31.73076923 | |
| 417 31.65467626 | |
| 418 31.57894737 | |
| 419 31.50357995 | |
| 420 31.42857143 | |
| 421 31.59144893 | |
| 422 31.51658768 | |
| 423 31.67848700 | |
| 424 31.60377358 | |
| 425 31.52941176 | |
| 426 31.69014085 | |
| 427 31.61592506 | |
| 428 31.54205607 | |
| 429 31.46853147 | |
| 430 31.39534884 | |
| 431 31.32250580 | |
| 432 31.48148148 | |
| 433 31.63972286 | |
| 434 31.79723502 | |
| 435 31.72413793 | |
| 436 31.88073394 | |
| 437 31.80778032 | |
| 438 31.96347032 | |
| 439 31.89066059 | |
| 440 32.04545455 | |
| 441 31.97278912 | |
| 442 32.12669683 | |
| 443 32.05417607 | |
| 444 32.20720721 | |
| 445 32.13483146 | |
| 446 32.06278027 | |
| 447 31.99105145 | |
| 448 31.91964286 | |
| 449 32.07126949 | |
| 450 32.22222222 | |
| 451 32.15077605 | |
| 452 32.07964602 | |
| 453 32.22958057 | |
| 454 32.15859031 | |
| 455 32.08791209 | |
| 456 32.01754386 | |
| 457 31.94748359 | |
| 458 31.87772926 | |
| 459 31.80827887 | |
| 460 31.73913043 | |
| 461 31.67028200 | |
| 462 31.81818182 | |
| 463 31.74946004 | |
| 464 31.89655172 | |
| 465 32.04301075 | |
| 466 31.97424893 | |
| 467 31.90578158 | |
| 468 31.83760684 | |
| 469 31.76972281 | |
| 470 31.70212766 | |
| 471 31.63481953 | |
| 472 31.77966102 | |
| 473 31.92389006 | |
| 474 31.85654008 | |
| 475 32.00000000 | |
| 476 31.93277311 | |
| 477 31.86582809 | |
| 478 31.79916318 | |
| 479 31.73277662 | |
| 480 31.66666667 | |
| 481 31.60083160 | |
| 482 31.74273859 | |
| 483 31.67701863 | |
| 484 31.61157025 | |
| 485 31.75257732 | |
| 486 31.89300412 | |
| 487 32.03285421 | |
| 488 31.96721311 | |
| 489 32.10633947 | |
| 490 32.24489796 | |
| 491 32.38289206 | |
| 492 32.52032520 | |
| 493 32.45436105 | |
| 494 32.38866397 | |
| 495 32.32323232 | |
| 496 32.25806452 | |
| 497 32.19315895 | |
| 498 32.12851406 | |
| 499 32.26452906 | |
| 500 32.20000000 | |
| 501 32.33532934 | |
| 502 32.27091633 | |
| 503 32.20675944 | |
| 504 32.34126984 | |
| 505 32.27722772 | |
| 506 32.21343874 | |
| 507 32.14990138 | |
| 508 32.08661417 | |
| 509 32.22003929 | |
| 510 32.15686275 | |
| 511 32.09393346 | |
| 512 32.03125000 | |
| 513 31.96881092 | |
| 514 31.90661479 | |
| 515 32.03883495 | |
| 516 32.17054264 | |
| 517 32.30174081 | |
| 518 32.23938224 | |
| 519 32.17726397 | |
| 520 32.11538462 | |
| 521 32.24568138 | |
| 522 32.18390805 | |
| 523 32.12237094 | |
| 524 32.06106870 | |
| 525 32.00000000 | |
| 526 31.93916350 | |
| 527 32.06831120 | |
| 528 32.00757576 | |
| 529 31.94706994 | |
| 530 31.88679245 | |
| 531 31.82674200 | |
| 532 31.76691729 | |
| 533 31.89493433 | |
| 534 31.83520599 | |
| 535 31.77570093 | |
| 536 31.71641791 | |
| 537 31.65735568 | |
| 538 31.59851301 | |
| 539 31.53988868 | |
| 540 31.48148148 | |
| 541 31.42329020 | |
| 542 31.36531365 | |
| 543 31.30755064 | |
| 544 31.25000000 | |
| 545 31.19266055 | |
| 546 31.31868132 | |
| 547 31.44424132 | |
| 548 31.56934307 | |
| 549 31.51183971 | |
| 550 31.45454545 | |
| 551 31.39745917 | |
| 552 31.52173913 | |
| 553 31.64556962 | |
| 554 31.58844765 | |
| 555 31.53153153 | |
| 556 31.47482014 | |
| 557 31.41831239 | |
| 558 31.36200717 | |
| 559 31.30590340 | |
| 560 31.42857143 | |
| 561 31.55080214 | |
| 562 31.49466192 | |
| 563 31.43872114 | |
| 564 31.56028369 | |
| 565 31.68141593 | |
| 566 31.80212014 | |
| 567 31.92239859 | |
| 568 32.04225352 | |
| 569 31.98594025 | |
| 570 31.92982456 | |
| 571 32.04903678 | |
| 572 31.99300699 | |
| 573 31.93717277 | |
| 574 32.05574913 | |
| 575 32.17391304 | |
| 576 32.29166667 | |
| 577 32.23570191 | |
| 578 32.17993080 | |
| 579 32.12435233 | |
| 580 32.24137931 | |
| 581 32.18588640 | |
| 582 32.30240550 | |
| 583 32.24699828 | |
| 584 32.36301370 | |
| 585 32.30769231 | |
| 586 32.42320819 | |
| 587 32.36797274 | |
| 588 32.31292517 | |
| 589 32.25806452 | |
| 590 32.37288136 | |
| 591 32.31810491 | |
| 592 32.26351351 | |
| 593 32.20910624 | |
| 594 32.32323232 | |
| 595 32.26890756 | |
| 596 32.21476510 | |
| 597 32.16080402 | |
| 598 32.10702341 | |
| 599 32.05342237 | |
| 600 32.00000000 | |
| 601 32.11314476 | |
| 602 32.22591362 | |
| 603 32.33830846 | |
| 604 32.28476821 | |
| 605 32.23140496 | |
| 606 32.34323432 | |
| 607 32.28995058 | |
| 608 32.23684211 | |
| 609 32.34811166 | |
| 610 32.29508197 | |
| 611 32.24222586 | |
| 612 32.35294118 | |
| 613 32.30016313 | |
| 614 32.24755700 | |
| 615 32.19512195 | |
| 616 32.30519481 | |
| 617 32.41491086 | |
| 618 32.36245955 | |
| 619 32.31017771 | |
| 620 32.25806452 | |
| 621 32.20611916 | |
| 622 32.31511254 | |
| 623 32.26324238 | |
| 624 32.21153846 | |
| 625 32.16000000 | |
| 626 32.10862620 | |
| 627 32.05741627 | |
| 628 32.16560510 | |
| 629 32.11446741 | |
| 630 32.06349206 | |
| 631 32.01267829 | |
| 632 31.96202532 | |
| 633 31.91153239 | |
| 634 31.86119874 | |
| 635 31.81102362 | |
| 636 31.76100629 | |
| 637 31.71114600 | |
| 638 31.66144201 | |
| 639 31.61189358 | |
| 640 31.71875000 | |
| 641 31.66926677 | |
| 642 31.77570093 | |
| 643 31.72628305 | |
| 644 31.67701863 | |
| 645 31.78294574 | |
| 646 31.73374613 | |
| 647 31.68469861 | |
| 648 31.79012346 | |
| 649 31.74114022 | |
| 650 31.84615385 | |
| 651 31.95084485 | |
| 652 32.05521472 | |
| 653 32.15926493 | |
| 654 32.11009174 | |
| 655 32.06106870 | |
| 656 32.16463415 | |
| 657 32.26788432 | |
| 658 32.21884498 | |
| 659 32.32169954 | |
| 660 32.27272727 | |
| 661 32.22390318 | |
| 662 32.17522659 | |
| 663 32.12669683 | |
| 664 32.07831325 | |
| 665 32.18045113 | |
| 666 32.28228228 | |
| 667 32.23388306 | |
| 668 32.18562874 | |
| 669 32.13751868 | |
| 670 32.23880597 | |
| 671 32.19076006 | |
| 672 32.14285714 | |
| 673 32.09509658 | |
| 674 32.04747774 | |
| 675 32.14814815 | |
| 676 32.24852071 | |
| 677 32.20088626 | |
| 678 32.15339233 | |
| 679 32.10603829 | |
| 680 32.05882353 | |
| 681 32.01174743 | |
| 682 32.11143695 | |
| 683 32.06442167 | |
| 684 32.16374269 | |
| 685 32.11678832 | |
| 686 32.06997085 | |
| 687 32.02328967 | |
| 688 32.12209302 | |
| 689 32.07547170 | |
| 690 32.02898551 | |
| 691 32.12735166 | |
| 692 32.22543353 | |
| 693 32.32323232 | |
| 694 32.27665706 | |
| 695 32.23021583 | |
| 696 32.32758621 | |
| 697 32.42467719 | |
| 698 32.37822350 | |
| 699 32.33190272 | |
| 700 32.42857143 | |
| 701 32.38231098 | |
| 702 32.33618234 | |
| 703 32.29018492 | |
| 704 32.24431818 | |
| 705 32.19858156 | |
| 706 32.29461756 | |
| 707 32.24893918 | |
| 708 32.20338983 | |
| 709 32.15796897 | |
| 710 32.25352113 | |
| 711 32.20815752 | |
| 712 32.16292135 | |
| 713 32.11781206 | |
| 714 32.07282913 | |
| 715 32.02797203 | |
| 716 31.98324022 | |
| 717 32.07810321 | |
| 718 32.17270195 | |
| 719 32.26703755 | |
| 720 32.22222222 | |
| 721 32.17753121 | |
| 722 32.13296399 | |
| 723 32.08852006 | |
| 724 32.18232044 | |
| 725 32.13793103 | |
| 726 32.23140496 | |
| 727 32.18707015 | |
| 728 32.14285714 | |
| 729 32.09876543 | |
| 730 32.19178082 | |
| 731 32.14774282 | |
| 732 32.24043716 | |
| 733 32.19645293 | |
| 734 32.15258856 | |
| 735 32.10884354 | |
| 736 32.06521739 | |
| 737 32.02170963 | |
| 738 31.97831978 | |
| 739 32.07036536 | |
| 740 32.16216216 | |
| 741 32.25371120 | |
| 742 32.21024259 | |
| 743 32.16689098 | |
| 744 32.25806452 | |
| 745 32.21476510 | |
| 746 32.30563003 | |
| 747 32.39625167 | |
| 748 32.35294118 | |
| 749 32.44325768 | |
| 750 32.53333333 | |
| Final result: 32.5333 +/- 1.7119 | |
| Random chance: 19.8992 +/- 1.4588 | |