| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 817 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 817 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 0.00000000 | |
| 2 0.00000000 | |
| 3 0.00000000 | |
| 4 0.00000000 | |
| 5 20.00000000 | |
| 6 16.66666667 | |
| 7 14.28571429 | |
| 8 25.00000000 | |
| 9 22.22222222 | |
| 10 20.00000000 | |
| 11 18.18181818 | |
| 12 25.00000000 | |
| 13 30.76923077 | |
| 14 28.57142857 | |
| 15 33.33333333 | |
| 16 37.50000000 | |
| 17 35.29411765 | |
| 18 33.33333333 | |
| 19 31.57894737 | |
| 20 30.00000000 | |
| 21 28.57142857 | |
| 22 27.27272727 | |
| 23 26.08695652 | |
| 24 29.16666667 | |
| 25 28.00000000 | |
| 26 26.92307692 | |
| 27 25.92592593 | |
| 28 25.00000000 | |
| 29 27.58620690 | |
| 30 30.00000000 | |
| 31 32.25806452 | |
| 32 34.37500000 | |
| 33 36.36363636 | |
| 34 35.29411765 | |
| 35 34.28571429 | |
| 36 33.33333333 | |
| 37 35.13513514 | |
| 38 34.21052632 | |
| 39 35.89743590 | |
| 40 35.00000000 | |
| 41 34.14634146 | |
| 42 33.33333333 | |
| 43 34.88372093 | |
| 44 36.36363636 | |
| 45 37.77777778 | |
| 46 36.95652174 | |
| 47 36.17021277 | |
| 48 37.50000000 | |
| 49 36.73469388 | |
| 50 38.00000000 | |
| 51 37.25490196 | |
| 52 36.53846154 | |
| 53 35.84905660 | |
| 54 37.03703704 | |
| 55 38.18181818 | |
| 56 37.50000000 | |
| 57 36.84210526 | |
| 58 36.20689655 | |
| 59 37.28813559 | |
| 60 38.33333333 | |
| 61 37.70491803 | |
| 62 38.70967742 | |
| 63 38.09523810 | |
| 64 37.50000000 | |
| 65 36.92307692 | |
| 66 36.36363636 | |
| 67 35.82089552 | |
| 68 35.29411765 | |
| 69 36.23188406 | |
| 70 35.71428571 | |
| 71 35.21126761 | |
| 72 36.11111111 | |
| 73 36.98630137 | |
| 74 36.48648649 | |
| 75 36.00000000 | |
| 76 36.84210526 | |
| 77 36.36363636 | |
| 78 35.89743590 | |
| 79 35.44303797 | |
| 80 36.25000000 | |
| 81 35.80246914 | |
| 82 36.58536585 | |
| 83 36.14457831 | |
| 84 35.71428571 | |
| 85 35.29411765 | |
| 86 34.88372093 | |
| 87 34.48275862 | |
| 88 34.09090909 | |
| 89 34.83146067 | |
| 90 34.44444444 | |
| 91 34.06593407 | |
| 92 33.69565217 | |
| 93 33.33333333 | |
| 94 32.97872340 | |
| 95 32.63157895 | |
| 96 33.33333333 | |
| 97 32.98969072 | |
| 98 32.65306122 | |
| 99 33.33333333 | |
| 100 33.00000000 | |
| 101 33.66336634 | |
| 102 33.33333333 | |
| 103 33.98058252 | |
| 104 33.65384615 | |
| 105 33.33333333 | |
| 106 33.01886792 | |
| 107 33.64485981 | |
| 108 34.25925926 | |
| 109 33.94495413 | |
| 110 34.54545455 | |
| 111 34.23423423 | |
| 112 34.82142857 | |
| 113 35.39823009 | |
| 114 35.08771930 | |
| 115 34.78260870 | |
| 116 34.48275862 | |
| 117 34.18803419 | |
| 118 33.89830508 | |
| 119 33.61344538 | |
| 120 34.16666667 | |
| 121 34.71074380 | |
| 122 34.42622951 | |
| 123 34.14634146 | |
| 124 34.67741935 | |
| 125 35.20000000 | |
| 126 35.71428571 | |
| 127 35.43307087 | |
| 128 35.15625000 | |
| 129 35.65891473 | |
| 130 35.38461538 | |
| 131 35.11450382 | |
| 132 34.84848485 | |
| 133 34.58646617 | |
| 134 34.32835821 | |
| 135 34.81481481 | |
| 136 34.55882353 | |
| 137 35.03649635 | |
| 138 35.50724638 | |
| 139 35.97122302 | |
| 140 35.71428571 | |
| 141 35.46099291 | |
| 142 35.21126761 | |
| 143 34.96503497 | |
| 144 35.41666667 | |
| 145 35.17241379 | |
| 146 34.93150685 | |
| 147 34.69387755 | |
| 148 34.45945946 | |
| 149 34.22818792 | |
| 150 34.00000000 | |
| 151 34.43708609 | |
| 152 34.21052632 | |
| 153 33.98692810 | |
| 154 33.76623377 | |
| 155 33.54838710 | |
| 156 33.97435897 | |
| 157 33.75796178 | |
| 158 34.17721519 | |
| 159 33.96226415 | |
| 160 33.75000000 | |
| 161 33.54037267 | |
| 162 33.95061728 | |
| 163 33.74233129 | |
| 164 33.53658537 | |
| 165 33.93939394 | |
| 166 34.33734940 | |
| 167 34.13173653 | |
| 168 34.52380952 | |
| 169 34.31952663 | |
| 170 34.11764706 | |
| 171 33.91812865 | |
| 172 33.72093023 | |
| 173 33.52601156 | |
| 174 33.90804598 | |
| 175 33.71428571 | |
| 176 34.09090909 | |
| 177 33.89830508 | |
| 178 33.70786517 | |
| 179 34.07821229 | |
| 180 33.88888889 | |
| 181 33.70165746 | |
| 182 33.51648352 | |
| 183 33.33333333 | |
| 184 33.15217391 | |
| 185 32.97297297 | |
| 186 33.33333333 | |
| 187 33.15508021 | |
| 188 32.97872340 | |
| 189 33.33333333 | |
| 190 33.15789474 | |
| 191 32.98429319 | |
| 192 32.81250000 | |
| 193 32.64248705 | |
| 194 32.98969072 | |
| 195 32.82051282 | |
| 196 32.65306122 | |
| 197 32.99492386 | |
| 198 32.82828283 | |
| 199 32.66331658 | |
| 200 33.00000000 | |
| 201 32.83582090 | |
| 202 32.67326733 | |
| 203 32.51231527 | |
| 204 32.35294118 | |
| 205 32.19512195 | |
| 206 32.03883495 | |
| 207 32.36714976 | |
| 208 32.21153846 | |
| 209 32.05741627 | |
| 210 31.90476190 | |
| 211 31.75355450 | |
| 212 31.60377358 | |
| 213 31.45539906 | |
| 214 31.30841121 | |
| 215 31.16279070 | |
| 216 31.48148148 | |
| 217 31.33640553 | |
| 218 31.65137615 | |
| 219 31.50684932 | |
| 220 31.36363636 | |
| 221 31.22171946 | |
| 222 31.08108108 | |
| 223 30.94170404 | |
| 224 31.25000000 | |
| 225 31.11111111 | |
| 226 30.97345133 | |
| 227 31.27753304 | |
| 228 31.14035088 | |
| 229 31.00436681 | |
| 230 30.86956522 | |
| 231 30.73593074 | |
| 232 30.60344828 | |
| 233 30.90128755 | |
| 234 31.19658120 | |
| 235 31.48936170 | |
| 236 31.77966102 | |
| 237 31.64556962 | |
| 238 31.93277311 | |
| 239 31.79916318 | |
| 240 31.66666667 | |
| 241 31.53526971 | |
| 242 31.40495868 | |
| 243 31.27572016 | |
| 244 31.14754098 | |
| 245 31.02040816 | |
| 246 30.89430894 | |
| 247 30.76923077 | |
| 248 30.64516129 | |
| 249 30.52208835 | |
| 250 30.40000000 | |
| 251 30.67729084 | |
| 252 30.95238095 | |
| 253 30.83003953 | |
| 254 30.70866142 | |
| 255 30.98039216 | |
| 256 30.85937500 | |
| 257 30.73929961 | |
| 258 30.62015504 | |
| 259 30.50193050 | |
| 260 30.38461538 | |
| 261 30.65134100 | |
| 262 30.53435115 | |
| 263 30.41825095 | |
| 264 30.68181818 | |
| 265 30.94339623 | |
| 266 30.82706767 | |
| 267 30.71161049 | |
| 268 30.59701493 | |
| 269 30.48327138 | |
| 270 30.37037037 | |
| 271 30.25830258 | |
| 272 30.14705882 | |
| 273 30.40293040 | |
| 274 30.29197080 | |
| 275 30.18181818 | |
| 276 30.07246377 | |
| 277 29.96389892 | |
| 278 30.21582734 | |
| 279 30.10752688 | |
| 280 30.35714286 | |
| 281 30.60498221 | |
| 282 30.49645390 | |
| 283 30.74204947 | |
| 284 30.63380282 | |
| 285 30.52631579 | |
| 286 30.76923077 | |
| 287 31.01045296 | |
| 288 31.25000000 | |
| 289 31.14186851 | |
| 290 31.03448276 | |
| 291 31.27147766 | |
| 292 31.50684932 | |
| 293 31.74061433 | |
| 294 31.63265306 | |
| 295 31.86440678 | |
| 296 32.09459459 | |
| 297 32.32323232 | |
| 298 32.21476510 | |
| 299 32.10702341 | |
| 300 32.00000000 | |
| 301 31.89368771 | |
| 302 31.78807947 | |
| 303 31.68316832 | |
| 304 31.57894737 | |
| 305 31.47540984 | |
| 306 31.37254902 | |
| 307 31.27035831 | |
| 308 31.49350649 | |
| 309 31.39158576 | |
| 310 31.29032258 | |
| 311 31.18971061 | |
| 312 31.41025641 | |
| 313 31.30990415 | |
| 314 31.21019108 | |
| 315 31.11111111 | |
| 316 31.01265823 | |
| 317 31.23028391 | |
| 318 31.13207547 | |
| 319 31.03448276 | |
| 320 30.93750000 | |
| 321 30.84112150 | |
| 322 31.05590062 | |
| 323 30.95975232 | |
| 324 30.86419753 | |
| 325 30.76923077 | |
| 326 30.67484663 | |
| 327 30.88685015 | |
| 328 31.09756098 | |
| 329 31.00303951 | |
| 330 31.21212121 | |
| 331 31.11782477 | |
| 332 31.32530120 | |
| 333 31.53153153 | |
| 334 31.43712575 | |
| 335 31.64179104 | |
| 336 31.84523810 | |
| 337 31.75074184 | |
| 338 31.65680473 | |
| 339 31.85840708 | |
| 340 31.76470588 | |
| 341 31.96480938 | |
| 342 32.16374269 | |
| 343 32.06997085 | |
| 344 32.26744186 | |
| 345 32.46376812 | |
| 346 32.36994220 | |
| 347 32.56484150 | |
| 348 32.47126437 | |
| 349 32.37822350 | |
| 350 32.28571429 | |
| 351 32.19373219 | |
| 352 32.38636364 | |
| 353 32.29461756 | |
| 354 32.20338983 | |
| 355 32.39436620 | |
| 356 32.30337079 | |
| 357 32.49299720 | |
| 358 32.40223464 | |
| 359 32.31197772 | |
| 360 32.22222222 | |
| 361 32.13296399 | |
| 362 32.04419890 | |
| 363 31.95592287 | |
| 364 31.86813187 | |
| 365 31.78082192 | |
| 366 31.69398907 | |
| 367 31.88010899 | |
| 368 31.79347826 | |
| 369 31.70731707 | |
| 370 31.62162162 | |
| 371 31.53638814 | |
| 372 31.72043011 | |
| 373 31.63538874 | |
| 374 31.81818182 | |
| 375 31.73333333 | |
| 376 31.64893617 | |
| 377 31.83023873 | |
| 378 31.74603175 | |
| 379 31.66226913 | |
| 380 31.84210526 | |
| 381 31.75853018 | |
| 382 31.93717277 | |
| 383 31.85378590 | |
| 384 31.77083333 | |
| 385 31.68831169 | |
| 386 31.60621762 | |
| 387 31.52454780 | |
| 388 31.70103093 | |
| 389 31.61953728 | |
| 390 31.53846154 | |
| 391 31.71355499 | |
| 392 31.63265306 | |
| 393 31.80661578 | |
| 394 31.97969543 | |
| 395 31.89873418 | |
| 396 32.07070707 | |
| 397 32.24181360 | |
| 398 32.16080402 | |
| 399 32.08020050 | |
| 400 32.00000000 | |
| 401 32.16957606 | |
| 402 32.33830846 | |
| 403 32.25806452 | |
| 404 32.17821782 | |
| 405 32.09876543 | |
| 406 32.01970443 | |
| 407 31.94103194 | |
| 408 31.86274510 | |
| 409 31.78484108 | |
| 410 31.70731707 | |
| 411 31.63017032 | |
| 412 31.79611650 | |
| 413 31.96125908 | |
| 414 32.12560386 | |
| 415 32.28915663 | |
| 416 32.21153846 | |
| 417 32.13429257 | |
| 418 32.05741627 | |
| 419 31.98090692 | |
| 420 32.14285714 | |
| 421 32.30403800 | |
| 422 32.22748815 | |
| 423 32.15130024 | |
| 424 32.07547170 | |
| 425 32.00000000 | |
| 426 31.92488263 | |
| 427 31.85011710 | |
| 428 31.77570093 | |
| 429 31.70163170 | |
| 430 31.62790698 | |
| 431 31.55452436 | |
| 432 31.71296296 | |
| 433 31.87066975 | |
| 434 32.02764977 | |
| 435 31.95402299 | |
| 436 32.11009174 | |
| 437 32.03661327 | |
| 438 32.19178082 | |
| 439 32.11845103 | |
| 440 32.27272727 | |
| 441 32.19954649 | |
| 442 32.12669683 | |
| 443 32.05417607 | |
| 444 32.20720721 | |
| 445 32.13483146 | |
| 446 32.06278027 | |
| 447 31.99105145 | |
| 448 31.91964286 | |
| 449 32.07126949 | |
| 450 32.22222222 | |
| 451 32.15077605 | |
| 452 32.07964602 | |
| 453 32.22958057 | |
| 454 32.15859031 | |
| 455 32.08791209 | |
| 456 32.01754386 | |
| 457 31.94748359 | |
| 458 31.87772926 | |
| 459 31.80827887 | |
| 460 31.73913043 | |
| 461 31.67028200 | |
| 462 31.81818182 | |
| 463 31.74946004 | |
| 464 31.89655172 | |
| 465 32.04301075 | |
| 466 31.97424893 | |
| 467 31.90578158 | |
| 468 32.05128205 | |
| 469 31.98294243 | |
| 470 31.91489362 | |
| 471 31.84713376 | |
| 472 31.99152542 | |
| 473 31.92389006 | |
| 474 31.85654008 | |
| 475 32.00000000 | |
| 476 31.93277311 | |
| 477 31.86582809 | |
| 478 31.79916318 | |
| 479 31.73277662 | |
| 480 31.66666667 | |
| 481 31.60083160 | |
| 482 31.74273859 | |
| 483 31.67701863 | |
| 484 31.81818182 | |
| 485 31.95876289 | |
| 486 31.89300412 | |
| 487 32.03285421 | |
| 488 31.96721311 | |
| 489 32.10633947 | |
| 490 32.04081633 | |
| 491 32.17922607 | |
| 492 32.31707317 | |
| 493 32.25152130 | |
| 494 32.38866397 | |
| 495 32.52525253 | |
| 496 32.45967742 | |
| 497 32.39436620 | |
| 498 32.32931727 | |
| 499 32.46492986 | |
| 500 32.40000000 | |
| 501 32.53493014 | |
| 502 32.47011952 | |
| 503 32.40556660 | |
| 504 32.53968254 | |
| 505 32.47524752 | |
| 506 32.41106719 | |
| 507 32.34714004 | |
| 508 32.48031496 | |
| 509 32.61296660 | |
| 510 32.74509804 | |
| 511 32.68101761 | |
| 512 32.61718750 | |
| 513 32.55360624 | |
| 514 32.49027237 | |
| 515 32.62135922 | |
| 516 32.75193798 | |
| 517 32.88201161 | |
| 518 32.81853282 | |
| 519 32.75529865 | |
| 520 32.69230769 | |
| 521 32.62955854 | |
| 522 32.56704981 | |
| 523 32.50478011 | |
| 524 32.44274809 | |
| 525 32.38095238 | |
| 526 32.31939163 | |
| 527 32.44781784 | |
| 528 32.38636364 | |
| 529 32.32514178 | |
| 530 32.26415094 | |
| 531 32.20338983 | |
| 532 32.33082707 | |
| 533 32.45778612 | |
| 534 32.39700375 | |
| 535 32.33644860 | |
| 536 32.27611940 | |
| 537 32.21601490 | |
| 538 32.15613383 | |
| 539 32.09647495 | |
| 540 32.03703704 | |
| 541 31.97781885 | |
| 542 31.91881919 | |
| 543 31.86003683 | |
| 544 31.80147059 | |
| 545 31.74311927 | |
| 546 31.86813187 | |
| 547 31.99268739 | |
| 548 32.11678832 | |
| 549 32.05828780 | |
| 550 32.00000000 | |
| 551 31.94192377 | |
| 552 32.06521739 | |
| 553 32.18806510 | |
| 554 32.12996390 | |
| 555 32.07207207 | |
| 556 32.01438849 | |
| 557 31.95691203 | |
| 558 31.89964158 | |
| 559 31.84257603 | |
| 560 31.96428571 | |
| 561 32.08556150 | |
| 562 32.02846975 | |
| 563 31.97158082 | |
| 564 32.09219858 | |
| 565 32.21238938 | |
| 566 32.33215548 | |
| 567 32.45149912 | |
| 568 32.57042254 | |
| 569 32.51318102 | |
| 570 32.45614035 | |
| 571 32.57443082 | |
| 572 32.51748252 | |
| 573 32.46073298 | |
| 574 32.57839721 | |
| 575 32.69565217 | |
| 576 32.81250000 | |
| 577 32.92894281 | |
| 578 33.04498270 | |
| 579 33.16062176 | |
| 580 33.27586207 | |
| 581 33.39070568 | |
| 582 33.50515464 | |
| 583 33.44768439 | |
| 584 33.56164384 | |
| 585 33.50427350 | |
| 586 33.61774744 | |
| 587 33.56047700 | |
| 588 33.50340136 | |
| 589 33.44651952 | |
| 590 33.55932203 | |
| 591 33.50253807 | |
| 592 33.44594595 | |
| 593 33.55817875 | |
| 594 33.67003367 | |
| 595 33.61344538 | |
| 596 33.55704698 | |
| 597 33.50083752 | |
| 598 33.44481605 | |
| 599 33.55592654 | |
| 600 33.50000000 | |
| 601 33.61064892 | |
| 602 33.72093023 | |
| 603 33.83084577 | |
| 604 33.77483444 | |
| 605 33.71900826 | |
| 606 33.82838284 | |
| 607 33.77265239 | |
| 608 33.71710526 | |
| 609 33.82594417 | |
| 610 33.77049180 | |
| 611 33.71522095 | |
| 612 33.82352941 | |
| 613 33.76835237 | |
| 614 33.71335505 | |
| 615 33.65853659 | |
| 616 33.60389610 | |
| 617 33.71150729 | |
| 618 33.65695793 | |
| 619 33.76413570 | |
| 620 33.70967742 | |
| 621 33.65539452 | |
| 622 33.76205788 | |
| 623 33.70786517 | |
| 624 33.65384615 | |
| 625 33.60000000 | |
| 626 33.54632588 | |
| 627 33.65231260 | |
| 628 33.75796178 | |
| 629 33.86327504 | |
| 630 33.80952381 | |
| 631 33.91442155 | |
| 632 33.86075949 | |
| 633 33.80726698 | |
| 634 33.91167192 | |
| 635 33.85826772 | |
| 636 33.80503145 | |
| 637 33.75196232 | |
| 638 33.69905956 | |
| 639 33.64632238 | |
| 640 33.75000000 | |
| 641 33.69734789 | |
| 642 33.80062305 | |
| 643 33.74805599 | |
| 644 33.69565217 | |
| 645 33.79844961 | |
| 646 33.74613003 | |
| 647 33.84853168 | |
| 648 33.95061728 | |
| 649 33.89830508 | |
| 650 34.00000000 | |
| 651 34.10138249 | |
| 652 34.20245399 | |
| 653 34.30321593 | |
| 654 34.25076453 | |
| 655 34.19847328 | |
| 656 34.29878049 | |
| 657 34.24657534 | |
| 658 34.19452888 | |
| 659 34.29438543 | |
| 660 34.24242424 | |
| 661 34.19062027 | |
| 662 34.13897281 | |
| 663 34.08748115 | |
| 664 34.03614458 | |
| 665 34.13533835 | |
| 666 34.23423423 | |
| 667 34.18290855 | |
| 668 34.13173653 | |
| 669 34.08071749 | |
| 670 34.17910448 | |
| 671 34.12816692 | |
| 672 34.07738095 | |
| 673 34.02674591 | |
| 674 33.97626113 | |
| 675 34.07407407 | |
| 676 34.17159763 | |
| 677 34.12112260 | |
| 678 34.07079646 | |
| 679 34.02061856 | |
| 680 33.97058824 | |
| 681 33.92070485 | |
| 682 33.87096774 | |
| 683 33.82137628 | |
| 684 33.91812865 | |
| 685 33.86861314 | |
| 686 33.81924198 | |
| 687 33.77001456 | |
| 688 33.72093023 | |
| 689 33.81712627 | |
| 690 33.76811594 | |
| 691 33.86396527 | |
| 692 33.95953757 | |
| 693 34.05483405 | |
| 694 34.00576369 | |
| 695 33.95683453 | |
| 696 34.05172414 | |
| 697 34.00286944 | |
| 698 33.95415473 | |
| 699 34.04864092 | |
| 700 34.14285714 | |
| 701 34.09415121 | |
| 702 34.04558405 | |
| 703 33.99715505 | |
| 704 33.94886364 | |
| 705 33.90070922 | |
| 706 33.99433428 | |
| 707 33.94625177 | |
| 708 33.89830508 | |
| 709 33.85049365 | |
| 710 33.80281690 | |
| 711 33.75527426 | |
| 712 33.70786517 | |
| 713 33.66058906 | |
| 714 33.75350140 | |
| 715 33.84615385 | |
| 716 33.79888268 | |
| 717 33.89121339 | |
| 718 33.98328691 | |
| 719 34.07510431 | |
| 720 34.02777778 | |
| 721 33.98058252 | |
| 722 33.93351801 | |
| 723 33.88658368 | |
| 724 33.97790055 | |
| 725 33.93103448 | |
| 726 34.02203857 | |
| 727 33.97524072 | |
| 728 33.92857143 | |
| 729 33.88203018 | |
| 730 33.97260274 | |
| 731 33.92612859 | |
| 732 34.01639344 | |
| 733 33.96998636 | |
| 734 33.92370572 | |
| 735 33.87755102 | |
| 736 33.83152174 | |
| 737 33.92130258 | |
| 738 33.87533875 | |
| 739 33.96481732 | |
| 740 33.91891892 | |
| 741 34.00809717 | |
| 742 33.96226415 | |
| 743 33.91655451 | |
| 744 34.00537634 | |
| 745 33.95973154 | |
| 746 34.04825737 | |
| 747 34.13654618 | |
| 748 34.09090909 | |
| 749 34.17890521 | |
| 750 34.26666667 | |
| Final result: 34.2667 +/- 1.7342 | |
| Random chance: 19.8992 +/- 1.4588 | |