| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 1548 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 1548 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 50.00000000 | |
| 5 40.00000000 | |
| 6 33.33333333 | |
| 7 42.85714286 | |
| 8 50.00000000 | |
| 9 44.44444444 | |
| 10 40.00000000 | |
| 11 36.36363636 | |
| 12 33.33333333 | |
| 13 38.46153846 | |
| 14 35.71428571 | |
| 15 40.00000000 | |
| 16 43.75000000 | |
| 17 41.17647059 | |
| 18 38.88888889 | |
| 19 42.10526316 | |
| 20 40.00000000 | |
| 21 38.09523810 | |
| 22 36.36363636 | |
| 23 34.78260870 | |
| 24 33.33333333 | |
| 25 36.00000000 | |
| 26 34.61538462 | |
| 27 33.33333333 | |
| 28 35.71428571 | |
| 29 37.93103448 | |
| 30 40.00000000 | |
| 31 38.70967742 | |
| 32 40.62500000 | |
| 33 39.39393939 | |
| 34 38.23529412 | |
| 35 40.00000000 | |
| 36 38.88888889 | |
| 37 37.83783784 | |
| 38 36.84210526 | |
| 39 35.89743590 | |
| 40 35.00000000 | |
| 41 34.14634146 | |
| 42 33.33333333 | |
| 43 32.55813953 | |
| 44 31.81818182 | |
| 45 31.11111111 | |
| 46 30.43478261 | |
| 47 29.78723404 | |
| 48 31.25000000 | |
| 49 32.65306122 | |
| 50 34.00000000 | |
| 51 33.33333333 | |
| 52 32.69230769 | |
| 53 32.07547170 | |
| 54 31.48148148 | |
| 55 32.72727273 | |
| 56 33.92857143 | |
| 57 35.08771930 | |
| 58 34.48275862 | |
| 59 33.89830508 | |
| 60 33.33333333 | |
| 61 32.78688525 | |
| 62 32.25806452 | |
| 63 31.74603175 | |
| 64 31.25000000 | |
| 65 32.30769231 | |
| 66 33.33333333 | |
| 67 32.83582090 | |
| 68 32.35294118 | |
| 69 31.88405797 | |
| 70 32.85714286 | |
| 71 32.39436620 | |
| 72 33.33333333 | |
| 73 32.87671233 | |
| 74 33.78378378 | |
| 75 34.66666667 | |
| 76 35.52631579 | |
| 77 36.36363636 | |
| 78 37.17948718 | |
| 79 36.70886076 | |
| 80 36.25000000 | |
| 81 37.03703704 | |
| 82 36.58536585 | |
| 83 37.34939759 | |
| 84 38.09523810 | |
| 85 37.64705882 | |
| 86 37.20930233 | |
| 87 37.93103448 | |
| 88 38.63636364 | |
| 89 38.20224719 | |
| 90 37.77777778 | |
| 91 37.36263736 | |
| 92 36.95652174 | |
| 93 36.55913978 | |
| 94 37.23404255 | |
| 95 36.84210526 | |
| 96 37.50000000 | |
| 97 37.11340206 | |
| 98 36.73469388 | |
| 99 36.36363636 | |
| 100 37.00000000 | |
| 101 36.63366337 | |
| 102 36.27450980 | |
| 103 35.92233010 | |
| 104 35.57692308 | |
| 105 36.19047619 | |
| 106 35.84905660 | |
| 107 35.51401869 | |
| 108 35.18518519 | |
| 109 34.86238532 | |
| 110 34.54545455 | |
| 111 34.23423423 | |
| 112 33.92857143 | |
| 113 33.62831858 | |
| 114 34.21052632 | |
| 115 34.78260870 | |
| 116 34.48275862 | |
| 117 34.18803419 | |
| 118 33.89830508 | |
| 119 33.61344538 | |
| 120 33.33333333 | |
| 121 33.05785124 | |
| 122 32.78688525 | |
| 123 32.52032520 | |
| 124 32.25806452 | |
| 125 32.80000000 | |
| 126 33.33333333 | |
| 127 33.07086614 | |
| 128 32.81250000 | |
| 129 32.55813953 | |
| 130 32.30769231 | |
| 131 32.82442748 | |
| 132 33.33333333 | |
| 133 33.08270677 | |
| 134 32.83582090 | |
| 135 32.59259259 | |
| 136 32.35294118 | |
| 137 32.11678832 | |
| 138 31.88405797 | |
| 139 31.65467626 | |
| 140 31.42857143 | |
| 141 31.91489362 | |
| 142 31.69014085 | |
| 143 32.16783217 | |
| 144 32.63888889 | |
| 145 32.41379310 | |
| 146 32.19178082 | |
| 147 31.97278912 | |
| 148 32.43243243 | |
| 149 32.21476510 | |
| 150 32.00000000 | |
| 151 31.78807947 | |
| 152 32.23684211 | |
| 153 32.02614379 | |
| 154 31.81818182 | |
| 155 31.61290323 | |
| 156 31.41025641 | |
| 157 31.84713376 | |
| 158 31.64556962 | |
| 159 31.44654088 | |
| 160 31.87500000 | |
| 161 31.67701863 | |
| 162 31.48148148 | |
| 163 31.28834356 | |
| 164 31.70731707 | |
| 165 32.12121212 | |
| 166 31.92771084 | |
| 167 32.33532934 | |
| 168 32.14285714 | |
| 169 31.95266272 | |
| 170 31.76470588 | |
| 171 31.57894737 | |
| 172 31.39534884 | |
| 173 31.21387283 | |
| 174 31.03448276 | |
| 175 30.85714286 | |
| 176 30.68181818 | |
| 177 30.50847458 | |
| 178 30.33707865 | |
| 179 30.72625698 | |
| 180 30.55555556 | |
| 181 30.38674033 | |
| 182 30.21978022 | |
| 183 30.60109290 | |
| 184 30.43478261 | |
| 185 30.27027027 | |
| 186 30.64516129 | |
| 187 30.48128342 | |
| 188 30.31914894 | |
| 189 30.15873016 | |
| 190 30.00000000 | |
| 191 30.36649215 | |
| 192 30.20833333 | |
| 193 30.05181347 | |
| 194 30.41237113 | |
| 195 30.76923077 | |
| 196 30.61224490 | |
| 197 30.45685279 | |
| 198 30.30303030 | |
| 199 30.15075377 | |
| 200 30.00000000 | |
| 201 30.34825871 | |
| 202 30.19801980 | |
| 203 30.04926108 | |
| 204 29.90196078 | |
| 205 30.24390244 | |
| 206 30.09708738 | |
| 207 29.95169082 | |
| 208 29.80769231 | |
| 209 29.66507177 | |
| 210 29.52380952 | |
| 211 29.85781991 | |
| 212 29.71698113 | |
| 213 29.57746479 | |
| 214 29.43925234 | |
| 215 29.30232558 | |
| 216 29.62962963 | |
| 217 29.49308756 | |
| 218 29.35779817 | |
| 219 29.22374429 | |
| 220 29.09090909 | |
| 221 28.95927602 | |
| 222 29.27927928 | |
| 223 29.14798206 | |
| 224 29.01785714 | |
| 225 29.33333333 | |
| 226 29.20353982 | |
| 227 29.51541850 | |
| 228 29.82456140 | |
| 229 29.69432314 | |
| 230 29.56521739 | |
| 231 29.43722944 | |
| 232 29.31034483 | |
| 233 29.18454936 | |
| 234 29.48717949 | |
| 235 29.36170213 | |
| 236 29.23728814 | |
| 237 29.11392405 | |
| 238 28.99159664 | |
| 239 28.87029289 | |
| 240 29.16666667 | |
| 241 29.04564315 | |
| 242 29.33884298 | |
| 243 29.62962963 | |
| 244 29.50819672 | |
| 245 29.79591837 | |
| 246 29.67479675 | |
| 247 29.55465587 | |
| 248 29.83870968 | |
| 249 29.71887550 | |
| 250 29.60000000 | |
| 251 29.48207171 | |
| 252 29.36507937 | |
| 253 29.24901186 | |
| 254 29.13385827 | |
| 255 29.41176471 | |
| 256 29.29687500 | |
| 257 29.57198444 | |
| 258 29.45736434 | |
| 259 29.34362934 | |
| 260 29.23076923 | |
| 261 29.11877395 | |
| 262 29.00763359 | |
| 263 28.89733840 | |
| 264 28.78787879 | |
| 265 28.67924528 | |
| 266 28.94736842 | |
| 267 28.83895131 | |
| 268 28.73134328 | |
| 269 28.62453532 | |
| 270 28.51851852 | |
| 271 28.41328413 | |
| 272 28.67647059 | |
| 273 28.57142857 | |
| 274 28.46715328 | |
| 275 28.36363636 | |
| 276 28.62318841 | |
| 277 28.88086643 | |
| 278 29.13669065 | |
| 279 29.03225806 | |
| 280 29.28571429 | |
| 281 29.18149466 | |
| 282 29.07801418 | |
| 283 28.97526502 | |
| 284 28.87323944 | |
| 285 28.77192982 | |
| 286 28.67132867 | |
| 287 28.91986063 | |
| 288 28.81944444 | |
| 289 28.71972318 | |
| 290 28.96551724 | |
| 291 29.20962199 | |
| 292 29.45205479 | |
| 293 29.69283276 | |
| 294 29.93197279 | |
| 295 29.83050847 | |
| 296 29.72972973 | |
| 297 29.62962963 | |
| 298 29.86577181 | |
| 299 29.76588629 | |
| 300 30.00000000 | |
| 301 30.23255814 | |
| 302 30.46357616 | |
| 303 30.36303630 | |
| 304 30.59210526 | |
| 305 30.81967213 | |
| 306 30.71895425 | |
| 307 30.61889251 | |
| 308 30.51948052 | |
| 309 30.42071197 | |
| 310 30.32258065 | |
| 311 30.54662379 | |
| 312 30.76923077 | |
| 313 30.99041534 | |
| 314 31.21019108 | |
| 315 31.11111111 | |
| 316 31.01265823 | |
| 317 30.91482650 | |
| 318 30.81761006 | |
| 319 31.03448276 | |
| 320 30.93750000 | |
| 321 30.84112150 | |
| 322 30.74534161 | |
| 323 30.65015480 | |
| 324 30.55555556 | |
| 325 30.76923077 | |
| 326 30.67484663 | |
| 327 30.88685015 | |
| 328 31.09756098 | |
| 329 31.00303951 | |
| 330 30.90909091 | |
| 331 30.81570997 | |
| 332 31.02409639 | |
| 333 30.93093093 | |
| 334 30.83832335 | |
| 335 31.04477612 | |
| 336 30.95238095 | |
| 337 30.86053412 | |
| 338 30.76923077 | |
| 339 30.67846608 | |
| 340 30.88235294 | |
| 341 30.79178886 | |
| 342 30.70175439 | |
| 343 30.90379009 | |
| 344 30.81395349 | |
| 345 30.72463768 | |
| 346 30.92485549 | |
| 347 31.12391931 | |
| 348 31.03448276 | |
| 349 30.94555874 | |
| 350 31.14285714 | |
| 351 31.05413105 | |
| 352 30.96590909 | |
| 353 31.16147309 | |
| 354 31.07344633 | |
| 355 30.98591549 | |
| 356 31.17977528 | |
| 357 31.09243697 | |
| 358 31.00558659 | |
| 359 30.91922006 | |
| 360 31.11111111 | |
| 361 31.30193906 | |
| 362 31.49171271 | |
| 363 31.68044077 | |
| 364 31.59340659 | |
| 365 31.50684932 | |
| 366 31.69398907 | |
| 367 31.60762943 | |
| 368 31.52173913 | |
| 369 31.43631436 | |
| 370 31.62162162 | |
| 371 31.53638814 | |
| 372 31.45161290 | |
| 373 31.36729223 | |
| 374 31.55080214 | |
| 375 31.73333333 | |
| 376 31.64893617 | |
| 377 31.56498674 | |
| 378 31.48148148 | |
| 379 31.39841689 | |
| 380 31.31578947 | |
| 381 31.49606299 | |
| 382 31.67539267 | |
| 383 31.85378590 | |
| 384 31.77083333 | |
| 385 31.68831169 | |
| 386 31.60621762 | |
| 387 31.52454780 | |
| 388 31.44329897 | |
| 389 31.61953728 | |
| 390 31.53846154 | |
| 391 31.71355499 | |
| 392 31.88775510 | |
| 393 31.80661578 | |
| 394 31.72588832 | |
| 395 31.64556962 | |
| 396 31.81818182 | |
| 397 31.73803526 | |
| 398 31.65829146 | |
| 399 31.82957393 | |
| 400 31.75000000 | |
| 401 31.92019950 | |
| 402 32.08955224 | |
| 403 32.00992556 | |
| 404 31.93069307 | |
| 405 31.85185185 | |
| 406 31.77339901 | |
| 407 31.69533170 | |
| 408 31.61764706 | |
| 409 31.78484108 | |
| 410 31.70731707 | |
| 411 31.63017032 | |
| 412 31.55339806 | |
| 413 31.71912833 | |
| 414 31.88405797 | |
| 415 32.04819277 | |
| 416 32.21153846 | |
| 417 32.13429257 | |
| 418 32.05741627 | |
| 419 32.21957041 | |
| 420 32.38095238 | |
| 421 32.30403800 | |
| 422 32.22748815 | |
| 423 32.15130024 | |
| 424 32.31132075 | |
| 425 32.23529412 | |
| 426 32.15962441 | |
| 427 32.08430913 | |
| 428 32.00934579 | |
| 429 32.16783217 | |
| 430 32.09302326 | |
| 431 32.25058005 | |
| 432 32.40740741 | |
| 433 32.33256351 | |
| 434 32.25806452 | |
| 435 32.18390805 | |
| 436 32.11009174 | |
| 437 32.03661327 | |
| 438 32.19178082 | |
| 439 32.34624146 | |
| 440 32.27272727 | |
| 441 32.19954649 | |
| 442 32.12669683 | |
| 443 32.05417607 | |
| 444 32.20720721 | |
| 445 32.13483146 | |
| 446 32.06278027 | |
| 447 31.99105145 | |
| 448 31.91964286 | |
| 449 31.84855234 | |
| 450 31.77777778 | |
| 451 31.70731707 | |
| 452 31.85840708 | |
| 453 32.00883002 | |
| 454 31.93832599 | |
| 455 31.86813187 | |
| 456 32.01754386 | |
| 457 32.16630197 | |
| 458 32.09606987 | |
| 459 32.24400871 | |
| 460 32.39130435 | |
| 461 32.32104121 | |
| 462 32.25108225 | |
| 463 32.39740821 | |
| 464 32.32758621 | |
| 465 32.25806452 | |
| 466 32.18884120 | |
| 467 32.33404711 | |
| 468 32.26495726 | |
| 469 32.40938166 | |
| 470 32.34042553 | |
| 471 32.27176221 | |
| 472 32.20338983 | |
| 473 32.34672304 | |
| 474 32.27848101 | |
| 475 32.21052632 | |
| 476 32.35294118 | |
| 477 32.28511530 | |
| 478 32.42677824 | |
| 479 32.56784969 | |
| 480 32.50000000 | |
| 481 32.43243243 | |
| 482 32.36514523 | |
| 483 32.29813665 | |
| 484 32.23140496 | |
| 485 32.16494845 | |
| 486 32.09876543 | |
| 487 32.23819302 | |
| 488 32.17213115 | |
| 489 32.10633947 | |
| 490 32.24489796 | |
| 491 32.38289206 | |
| 492 32.31707317 | |
| 493 32.25152130 | |
| 494 32.38866397 | |
| 495 32.52525253 | |
| 496 32.66129032 | |
| 497 32.59557344 | |
| 498 32.53012048 | |
| 499 32.46492986 | |
| 500 32.60000000 | |
| 501 32.53493014 | |
| 502 32.66932271 | |
| 503 32.60437376 | |
| 504 32.73809524 | |
| 505 32.67326733 | |
| 506 32.80632411 | |
| 507 32.74161736 | |
| 508 32.67716535 | |
| 509 32.61296660 | |
| 510 32.74509804 | |
| 511 32.68101761 | |
| 512 32.61718750 | |
| 513 32.55360624 | |
| 514 32.49027237 | |
| 515 32.42718447 | |
| 516 32.36434109 | |
| 517 32.49516441 | |
| 518 32.62548263 | |
| 519 32.56262042 | |
| 520 32.50000000 | |
| 521 32.43761996 | |
| 522 32.37547893 | |
| 523 32.50478011 | |
| 524 32.63358779 | |
| 525 32.57142857 | |
| 526 32.50950570 | |
| 527 32.63757116 | |
| 528 32.76515152 | |
| 529 32.89224953 | |
| 530 33.01886792 | |
| 531 32.95668550 | |
| 532 32.89473684 | |
| 533 33.02063790 | |
| 534 32.95880150 | |
| 535 33.08411215 | |
| 536 33.02238806 | |
| 537 33.14711359 | |
| 538 33.27137546 | |
| 539 33.20964750 | |
| 540 33.14814815 | |
| 541 33.08687616 | |
| 542 33.21033210 | |
| 543 33.14917127 | |
| 544 33.08823529 | |
| 545 33.02752294 | |
| 546 32.96703297 | |
| 547 32.90676417 | |
| 548 33.02919708 | |
| 549 32.96903461 | |
| 550 32.90909091 | |
| 551 33.03085299 | |
| 552 32.97101449 | |
| 553 32.91139241 | |
| 554 32.85198556 | |
| 555 32.79279279 | |
| 556 32.73381295 | |
| 557 32.67504488 | |
| 558 32.79569892 | |
| 559 32.73703041 | |
| 560 32.67857143 | |
| 561 32.62032086 | |
| 562 32.56227758 | |
| 563 32.68206039 | |
| 564 32.62411348 | |
| 565 32.74336283 | |
| 566 32.68551237 | |
| 567 32.62786596 | |
| 568 32.74647887 | |
| 569 32.68892794 | |
| 570 32.63157895 | |
| 571 32.74956217 | |
| 572 32.69230769 | |
| 573 32.63525305 | |
| 574 32.75261324 | |
| 575 32.69565217 | |
| 576 32.81250000 | |
| 577 32.92894281 | |
| 578 32.87197232 | |
| 579 32.81519862 | |
| 580 32.75862069 | |
| 581 32.70223752 | |
| 582 32.64604811 | |
| 583 32.76157804 | |
| 584 32.87671233 | |
| 585 32.82051282 | |
| 586 32.76450512 | |
| 587 32.70868825 | |
| 588 32.65306122 | |
| 589 32.59762309 | |
| 590 32.54237288 | |
| 591 32.48730964 | |
| 592 32.43243243 | |
| 593 32.37774030 | |
| 594 32.49158249 | |
| 595 32.43697479 | |
| 596 32.55033557 | |
| 597 32.49581240 | |
| 598 32.44147157 | |
| 599 32.55425710 | |
| 600 32.50000000 | |
| 601 32.61231281 | |
| 602 32.55813953 | |
| 603 32.66998342 | |
| 604 32.61589404 | |
| 605 32.72727273 | |
| 606 32.67326733 | |
| 607 32.78418451 | |
| 608 32.73026316 | |
| 609 32.67651888 | |
| 610 32.62295082 | |
| 611 32.56955810 | |
| 612 32.51633987 | |
| 613 32.46329527 | |
| 614 32.57328990 | |
| 615 32.52032520 | |
| 616 32.62987013 | |
| 617 32.57698541 | |
| 618 32.52427184 | |
| 619 32.47172859 | |
| 620 32.41935484 | |
| 621 32.36714976 | |
| 622 32.31511254 | |
| 623 32.26324238 | |
| 624 32.21153846 | |
| 625 32.16000000 | |
| 626 32.26837061 | |
| 627 32.21690590 | |
| 628 32.16560510 | |
| 629 32.27344992 | |
| 630 32.38095238 | |
| 631 32.32963550 | |
| 632 32.43670886 | |
| 633 32.38546603 | |
| 634 32.33438486 | |
| 635 32.28346457 | |
| 636 32.23270440 | |
| 637 32.33908948 | |
| 638 32.28840125 | |
| 639 32.23787167 | |
| 640 32.34375000 | |
| 641 32.29329173 | |
| 642 32.39875389 | |
| 643 32.34836703 | |
| 644 32.45341615 | |
| 645 32.40310078 | |
| 646 32.35294118 | |
| 647 32.30293663 | |
| 648 32.25308642 | |
| 649 32.20338983 | |
| 650 32.15384615 | |
| 651 32.10445469 | |
| 652 32.20858896 | |
| 653 32.15926493 | |
| 654 32.26299694 | |
| 655 32.21374046 | |
| 656 32.31707317 | |
| 657 32.26788432 | |
| 658 32.37082067 | |
| 659 32.32169954 | |
| 660 32.27272727 | |
| 661 32.22390318 | |
| 662 32.17522659 | |
| 663 32.12669683 | |
| 664 32.07831325 | |
| 665 32.03007519 | |
| 666 31.98198198 | |
| 667 31.93403298 | |
| 668 32.03592814 | |
| 669 31.98804185 | |
| 670 32.08955224 | |
| 671 32.04172876 | |
| 672 31.99404762 | |
| 673 31.94650817 | |
| 674 31.89910979 | |
| 675 31.85185185 | |
| 676 31.80473373 | |
| 677 31.75775480 | |
| 678 31.71091445 | |
| 679 31.66421208 | |
| 680 31.76470588 | |
| 681 31.86490455 | |
| 682 31.81818182 | |
| 683 31.91800878 | |
| 684 31.87134503 | |
| 685 31.82481752 | |
| 686 31.77842566 | |
| 687 31.87772926 | |
| 688 31.83139535 | |
| 689 31.78519594 | |
| 690 31.73913043 | |
| 691 31.83791606 | |
| 692 31.79190751 | |
| 693 31.74603175 | |
| 694 31.84438040 | |
| 695 31.79856115 | |
| 696 31.89655172 | |
| 697 31.85078910 | |
| 698 31.80515759 | |
| 699 31.90271817 | |
| 700 31.85714286 | |
| 701 31.95435093 | |
| 702 32.05128205 | |
| 703 32.14793741 | |
| 704 32.10227273 | |
| 705 32.05673759 | |
| 706 32.01133144 | |
| 707 31.96605375 | |
| 708 31.92090395 | |
| 709 31.87588152 | |
| 710 31.97183099 | |
| 711 32.06751055 | |
| 712 32.02247191 | |
| 713 32.11781206 | |
| 714 32.07282913 | |
| 715 32.02797203 | |
| 716 32.12290503 | |
| 717 32.07810321 | |
| 718 32.03342618 | |
| 719 31.98887344 | |
| 720 31.94444444 | |
| 721 31.90013870 | |
| 722 31.85595568 | |
| 723 31.95020747 | |
| 724 31.90607735 | |
| 725 32.00000000 | |
| 726 31.95592287 | |
| 727 31.91196699 | |
| 728 31.86813187 | |
| 729 31.82441701 | |
| 730 31.78082192 | |
| 731 31.73734610 | |
| 732 31.83060109 | |
| 733 31.92360164 | |
| 734 32.01634877 | |
| 735 31.97278912 | |
| 736 31.92934783 | |
| 737 32.02170963 | |
| 738 32.11382114 | |
| 739 32.20568336 | |
| 740 32.29729730 | |
| 741 32.25371120 | |
| 742 32.34501348 | |
| 743 32.43606999 | |
| 744 32.52688172 | |
| 745 32.48322148 | |
| 746 32.43967828 | |
| 747 32.39625167 | |
| 748 32.35294118 | |
| 749 32.30974633 | |
| 750 32.40000000 | |
| Final result: 32.4000 ±1.7100 | |
| Random chance: 25.0000 ±1.5822 | |