| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 1548 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 1548 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 50.00000000 | |
| 5 40.00000000 | |
| 6 33.33333333 | |
| 7 42.85714286 | |
| 8 37.50000000 | |
| 9 33.33333333 | |
| 10 30.00000000 | |
| 11 27.27272727 | |
| 12 25.00000000 | |
| 13 30.76923077 | |
| 14 28.57142857 | |
| 15 33.33333333 | |
| 16 37.50000000 | |
| 17 35.29411765 | |
| 18 33.33333333 | |
| 19 36.84210526 | |
| 20 35.00000000 | |
| 21 33.33333333 | |
| 22 31.81818182 | |
| 23 30.43478261 | |
| 24 29.16666667 | |
| 25 32.00000000 | |
| 26 30.76923077 | |
| 27 29.62962963 | |
| 28 32.14285714 | |
| 29 34.48275862 | |
| 30 36.66666667 | |
| 31 38.70967742 | |
| 32 40.62500000 | |
| 33 39.39393939 | |
| 34 38.23529412 | |
| 35 40.00000000 | |
| 36 38.88888889 | |
| 37 37.83783784 | |
| 38 36.84210526 | |
| 39 35.89743590 | |
| 40 35.00000000 | |
| 41 34.14634146 | |
| 42 33.33333333 | |
| 43 32.55813953 | |
| 44 31.81818182 | |
| 45 31.11111111 | |
| 46 30.43478261 | |
| 47 29.78723404 | |
| 48 31.25000000 | |
| 49 32.65306122 | |
| 50 34.00000000 | |
| 51 33.33333333 | |
| 52 32.69230769 | |
| 53 32.07547170 | |
| 54 31.48148148 | |
| 55 32.72727273 | |
| 56 33.92857143 | |
| 57 35.08771930 | |
| 58 34.48275862 | |
| 59 33.89830508 | |
| 60 33.33333333 | |
| 61 32.78688525 | |
| 62 32.25806452 | |
| 63 31.74603175 | |
| 64 31.25000000 | |
| 65 32.30769231 | |
| 66 33.33333333 | |
| 67 32.83582090 | |
| 68 32.35294118 | |
| 69 31.88405797 | |
| 70 32.85714286 | |
| 71 32.39436620 | |
| 72 33.33333333 | |
| 73 32.87671233 | |
| 74 32.43243243 | |
| 75 33.33333333 | |
| 76 34.21052632 | |
| 77 35.06493506 | |
| 78 35.89743590 | |
| 79 35.44303797 | |
| 80 36.25000000 | |
| 81 37.03703704 | |
| 82 36.58536585 | |
| 83 37.34939759 | |
| 84 38.09523810 | |
| 85 37.64705882 | |
| 86 37.20930233 | |
| 87 37.93103448 | |
| 88 38.63636364 | |
| 89 38.20224719 | |
| 90 37.77777778 | |
| 91 37.36263736 | |
| 92 36.95652174 | |
| 93 36.55913978 | |
| 94 36.17021277 | |
| 95 35.78947368 | |
| 96 36.45833333 | |
| 97 36.08247423 | |
| 98 35.71428571 | |
| 99 35.35353535 | |
| 100 36.00000000 | |
| 101 35.64356436 | |
| 102 35.29411765 | |
| 103 34.95145631 | |
| 104 34.61538462 | |
| 105 35.23809524 | |
| 106 34.90566038 | |
| 107 34.57943925 | |
| 108 34.25925926 | |
| 109 33.94495413 | |
| 110 33.63636364 | |
| 111 33.33333333 | |
| 112 33.03571429 | |
| 113 32.74336283 | |
| 114 33.33333333 | |
| 115 33.91304348 | |
| 116 33.62068966 | |
| 117 33.33333333 | |
| 118 33.05084746 | |
| 119 32.77310924 | |
| 120 32.50000000 | |
| 121 32.23140496 | |
| 122 31.96721311 | |
| 123 31.70731707 | |
| 124 31.45161290 | |
| 125 32.00000000 | |
| 126 31.74603175 | |
| 127 31.49606299 | |
| 128 31.25000000 | |
| 129 31.00775194 | |
| 130 30.76923077 | |
| 131 31.29770992 | |
| 132 31.81818182 | |
| 133 31.57894737 | |
| 134 31.34328358 | |
| 135 31.11111111 | |
| 136 30.88235294 | |
| 137 30.65693431 | |
| 138 30.43478261 | |
| 139 30.21582734 | |
| 140 30.00000000 | |
| 141 30.49645390 | |
| 142 30.28169014 | |
| 143 30.76923077 | |
| 144 31.25000000 | |
| 145 31.03448276 | |
| 146 30.82191781 | |
| 147 30.61224490 | |
| 148 31.08108108 | |
| 149 30.87248322 | |
| 150 30.66666667 | |
| 151 30.46357616 | |
| 152 30.26315789 | |
| 153 30.06535948 | |
| 154 29.87012987 | |
| 155 29.67741935 | |
| 156 29.48717949 | |
| 157 29.93630573 | |
| 158 29.74683544 | |
| 159 29.55974843 | |
| 160 30.00000000 | |
| 161 29.81366460 | |
| 162 29.62962963 | |
| 163 29.44785276 | |
| 164 29.26829268 | |
| 165 29.69696970 | |
| 166 29.51807229 | |
| 167 29.94011976 | |
| 168 29.76190476 | |
| 169 29.58579882 | |
| 170 29.41176471 | |
| 171 29.23976608 | |
| 172 29.06976744 | |
| 173 28.90173410 | |
| 174 28.73563218 | |
| 175 28.57142857 | |
| 176 28.40909091 | |
| 177 28.24858757 | |
| 178 28.08988764 | |
| 179 28.49162011 | |
| 180 28.33333333 | |
| 181 28.17679558 | |
| 182 28.02197802 | |
| 183 27.86885246 | |
| 184 27.71739130 | |
| 185 27.56756757 | |
| 186 27.95698925 | |
| 187 27.80748663 | |
| 188 27.65957447 | |
| 189 27.51322751 | |
| 190 27.36842105 | |
| 191 27.74869110 | |
| 192 27.60416667 | |
| 193 27.46113990 | |
| 194 27.83505155 | |
| 195 28.20512821 | |
| 196 28.06122449 | |
| 197 27.91878173 | |
| 198 27.77777778 | |
| 199 27.63819095 | |
| 200 27.50000000 | |
| 201 27.86069652 | |
| 202 27.72277228 | |
| 203 27.58620690 | |
| 204 27.45098039 | |
| 205 27.80487805 | |
| 206 27.66990291 | |
| 207 27.53623188 | |
| 208 27.40384615 | |
| 209 27.27272727 | |
| 210 27.14285714 | |
| 211 27.48815166 | |
| 212 27.35849057 | |
| 213 27.23004695 | |
| 214 27.10280374 | |
| 215 26.97674419 | |
| 216 27.31481481 | |
| 217 27.18894009 | |
| 218 27.06422018 | |
| 219 26.94063927 | |
| 220 26.81818182 | |
| 221 26.69683258 | |
| 222 27.02702703 | |
| 223 26.90582960 | |
| 224 26.78571429 | |
| 225 27.11111111 | |
| 226 26.99115044 | |
| 227 27.31277533 | |
| 228 27.63157895 | |
| 229 27.51091703 | |
| 230 27.39130435 | |
| 231 27.27272727 | |
| 232 27.58620690 | |
| 233 27.46781116 | |
| 234 27.77777778 | |
| 235 27.65957447 | |
| 236 27.54237288 | |
| 237 27.42616034 | |
| 238 27.31092437 | |
| 239 27.19665272 | |
| 240 27.50000000 | |
| 241 27.38589212 | |
| 242 27.68595041 | |
| 243 27.98353909 | |
| 244 27.86885246 | |
| 245 28.16326531 | |
| 246 28.04878049 | |
| 247 27.93522267 | |
| 248 28.22580645 | |
| 249 28.11244980 | |
| 250 28.00000000 | |
| 251 27.88844622 | |
| 252 27.77777778 | |
| 253 27.66798419 | |
| 254 27.55905512 | |
| 255 27.84313725 | |
| 256 27.73437500 | |
| 257 28.01556420 | |
| 258 27.90697674 | |
| 259 27.79922780 | |
| 260 27.69230769 | |
| 261 27.58620690 | |
| 262 27.48091603 | |
| 263 27.37642586 | |
| 264 27.27272727 | |
| 265 27.16981132 | |
| 266 27.06766917 | |
| 267 26.96629213 | |
| 268 26.86567164 | |
| 269 26.76579926 | |
| 270 26.66666667 | |
| 271 26.56826568 | |
| 272 26.83823529 | |
| 273 26.73992674 | |
| 274 26.64233577 | |
| 275 26.54545455 | |
| 276 26.81159420 | |
| 277 27.07581227 | |
| 278 27.33812950 | |
| 279 27.24014337 | |
| 280 27.50000000 | |
| 281 27.40213523 | |
| 282 27.30496454 | |
| 283 27.20848057 | |
| 284 27.11267606 | |
| 285 27.01754386 | |
| 286 26.92307692 | |
| 287 27.17770035 | |
| 288 27.08333333 | |
| 289 26.98961938 | |
| 290 27.24137931 | |
| 291 27.49140893 | |
| 292 27.73972603 | |
| 293 27.98634812 | |
| 294 28.23129252 | |
| 295 28.13559322 | |
| 296 28.04054054 | |
| 297 27.94612795 | |
| 298 28.18791946 | |
| 299 28.42809365 | |
| 300 28.66666667 | |
| 301 28.90365449 | |
| 302 29.13907285 | |
| 303 29.04290429 | |
| 304 29.27631579 | |
| 305 29.50819672 | |
| 306 29.41176471 | |
| 307 29.31596091 | |
| 308 29.22077922 | |
| 309 29.12621359 | |
| 310 29.03225806 | |
| 311 29.26045016 | |
| 312 29.48717949 | |
| 313 29.71246006 | |
| 314 29.93630573 | |
| 315 29.84126984 | |
| 316 29.74683544 | |
| 317 29.65299685 | |
| 318 29.55974843 | |
| 319 29.78056426 | |
| 320 29.68750000 | |
| 321 29.59501558 | |
| 322 29.50310559 | |
| 323 29.41176471 | |
| 324 29.32098765 | |
| 325 29.53846154 | |
| 326 29.44785276 | |
| 327 29.66360856 | |
| 328 29.87804878 | |
| 329 29.78723404 | |
| 330 29.69696970 | |
| 331 29.60725076 | |
| 332 29.81927711 | |
| 333 29.72972973 | |
| 334 29.64071856 | |
| 335 29.85074627 | |
| 336 29.76190476 | |
| 337 29.67359050 | |
| 338 29.58579882 | |
| 339 29.49852507 | |
| 340 29.70588235 | |
| 341 29.91202346 | |
| 342 29.82456140 | |
| 343 30.02915452 | |
| 344 29.94186047 | |
| 345 29.85507246 | |
| 346 30.05780347 | |
| 347 30.25936599 | |
| 348 30.17241379 | |
| 349 30.37249284 | |
| 350 30.57142857 | |
| 351 30.48433048 | |
| 352 30.39772727 | |
| 353 30.59490085 | |
| 354 30.50847458 | |
| 355 30.42253521 | |
| 356 30.61797753 | |
| 357 30.53221289 | |
| 358 30.44692737 | |
| 359 30.36211699 | |
| 360 30.55555556 | |
| 361 30.74792244 | |
| 362 30.93922652 | |
| 363 31.12947658 | |
| 364 31.04395604 | |
| 365 30.95890411 | |
| 366 30.87431694 | |
| 367 30.79019074 | |
| 368 30.70652174 | |
| 369 30.62330623 | |
| 370 30.81081081 | |
| 371 30.72776280 | |
| 372 30.64516129 | |
| 373 30.56300268 | |
| 374 30.74866310 | |
| 375 30.93333333 | |
| 376 30.85106383 | |
| 377 30.76923077 | |
| 378 30.68783069 | |
| 379 30.60686016 | |
| 380 30.52631579 | |
| 381 30.70866142 | |
| 382 30.62827225 | |
| 383 30.80939948 | |
| 384 30.72916667 | |
| 385 30.90909091 | |
| 386 30.82901554 | |
| 387 30.74935401 | |
| 388 30.67010309 | |
| 389 30.84832905 | |
| 390 30.76923077 | |
| 391 30.94629156 | |
| 392 31.12244898 | |
| 393 31.04325700 | |
| 394 30.96446701 | |
| 395 30.88607595 | |
| 396 31.06060606 | |
| 397 30.98236776 | |
| 398 30.90452261 | |
| 399 31.07769424 | |
| 400 31.00000000 | |
| 401 31.17206983 | |
| 402 31.34328358 | |
| 403 31.26550868 | |
| 404 31.18811881 | |
| 405 31.11111111 | |
| 406 31.03448276 | |
| 407 30.95823096 | |
| 408 30.88235294 | |
| 409 31.05134474 | |
| 410 30.97560976 | |
| 411 30.90024331 | |
| 412 30.82524272 | |
| 413 30.99273608 | |
| 414 31.15942029 | |
| 415 31.32530120 | |
| 416 31.49038462 | |
| 417 31.41486811 | |
| 418 31.33971292 | |
| 419 31.50357995 | |
| 420 31.66666667 | |
| 421 31.59144893 | |
| 422 31.51658768 | |
| 423 31.44208038 | |
| 424 31.60377358 | |
| 425 31.76470588 | |
| 426 31.92488263 | |
| 427 31.85011710 | |
| 428 31.77570093 | |
| 429 31.93473193 | |
| 430 31.86046512 | |
| 431 32.01856148 | |
| 432 32.17592593 | |
| 433 32.10161663 | |
| 434 32.25806452 | |
| 435 32.18390805 | |
| 436 32.11009174 | |
| 437 32.03661327 | |
| 438 32.19178082 | |
| 439 32.34624146 | |
| 440 32.27272727 | |
| 441 32.19954649 | |
| 442 32.12669683 | |
| 443 32.05417607 | |
| 444 31.98198198 | |
| 445 31.91011236 | |
| 446 31.83856502 | |
| 447 31.76733781 | |
| 448 31.69642857 | |
| 449 31.62583519 | |
| 450 31.55555556 | |
| 451 31.48558758 | |
| 452 31.63716814 | |
| 453 31.78807947 | |
| 454 31.71806167 | |
| 455 31.64835165 | |
| 456 31.79824561 | |
| 457 31.94748359 | |
| 458 31.87772926 | |
| 459 32.02614379 | |
| 460 32.17391304 | |
| 461 32.10412148 | |
| 462 32.03463203 | |
| 463 32.18142549 | |
| 464 32.32758621 | |
| 465 32.25806452 | |
| 466 32.18884120 | |
| 467 32.33404711 | |
| 468 32.26495726 | |
| 469 32.40938166 | |
| 470 32.34042553 | |
| 471 32.27176221 | |
| 472 32.20338983 | |
| 473 32.34672304 | |
| 474 32.27848101 | |
| 475 32.21052632 | |
| 476 32.35294118 | |
| 477 32.28511530 | |
| 478 32.42677824 | |
| 479 32.56784969 | |
| 480 32.50000000 | |
| 481 32.64033264 | |
| 482 32.57261411 | |
| 483 32.50517598 | |
| 484 32.43801653 | |
| 485 32.37113402 | |
| 486 32.30452675 | |
| 487 32.44353183 | |
| 488 32.37704918 | |
| 489 32.31083845 | |
| 490 32.44897959 | |
| 491 32.58655804 | |
| 492 32.52032520 | |
| 493 32.45436105 | |
| 494 32.59109312 | |
| 495 32.72727273 | |
| 496 32.86290323 | |
| 497 32.79678068 | |
| 498 32.73092369 | |
| 499 32.66533066 | |
| 500 32.80000000 | |
| 501 32.73453094 | |
| 502 32.66932271 | |
| 503 32.60437376 | |
| 504 32.73809524 | |
| 505 32.67326733 | |
| 506 32.80632411 | |
| 507 32.74161736 | |
| 508 32.67716535 | |
| 509 32.61296660 | |
| 510 32.54901961 | |
| 511 32.48532290 | |
| 512 32.42187500 | |
| 513 32.35867446 | |
| 514 32.29571984 | |
| 515 32.23300971 | |
| 516 32.17054264 | |
| 517 32.30174081 | |
| 518 32.43243243 | |
| 519 32.36994220 | |
| 520 32.30769231 | |
| 521 32.24568138 | |
| 522 32.18390805 | |
| 523 32.31357553 | |
| 524 32.44274809 | |
| 525 32.38095238 | |
| 526 32.31939163 | |
| 527 32.44781784 | |
| 528 32.38636364 | |
| 529 32.51417769 | |
| 530 32.64150943 | |
| 531 32.58003766 | |
| 532 32.51879699 | |
| 533 32.64540338 | |
| 534 32.58426966 | |
| 535 32.71028037 | |
| 536 32.64925373 | |
| 537 32.77467412 | |
| 538 32.89962825 | |
| 539 32.83858998 | |
| 540 32.77777778 | |
| 541 32.71719039 | |
| 542 32.84132841 | |
| 543 32.78084715 | |
| 544 32.72058824 | |
| 545 32.66055046 | |
| 546 32.60073260 | |
| 547 32.54113346 | |
| 548 32.66423358 | |
| 549 32.60473588 | |
| 550 32.54545455 | |
| 551 32.66787659 | |
| 552 32.60869565 | |
| 553 32.54972875 | |
| 554 32.49097473 | |
| 555 32.43243243 | |
| 556 32.37410072 | |
| 557 32.31597846 | |
| 558 32.43727599 | |
| 559 32.37924866 | |
| 560 32.32142857 | |
| 561 32.26381462 | |
| 562 32.20640569 | |
| 563 32.32682060 | |
| 564 32.26950355 | |
| 565 32.38938053 | |
| 566 32.33215548 | |
| 567 32.27513228 | |
| 568 32.39436620 | |
| 569 32.33743409 | |
| 570 32.28070175 | |
| 571 32.39929947 | |
| 572 32.34265734 | |
| 573 32.28621291 | |
| 574 32.40418118 | |
| 575 32.34782609 | |
| 576 32.46527778 | |
| 577 32.58232236 | |
| 578 32.52595156 | |
| 579 32.46977547 | |
| 580 32.41379310 | |
| 581 32.35800344 | |
| 582 32.30240550 | |
| 583 32.41852487 | |
| 584 32.53424658 | |
| 585 32.47863248 | |
| 586 32.42320819 | |
| 587 32.36797274 | |
| 588 32.31292517 | |
| 589 32.25806452 | |
| 590 32.20338983 | |
| 591 32.14890017 | |
| 592 32.09459459 | |
| 593 32.04047218 | |
| 594 32.15488215 | |
| 595 32.10084034 | |
| 596 32.21476510 | |
| 597 32.16080402 | |
| 598 32.10702341 | |
| 599 32.22036728 | |
| 600 32.16666667 | |
| 601 32.27953411 | |
| 602 32.22591362 | |
| 603 32.33830846 | |
| 604 32.28476821 | |
| 605 32.23140496 | |
| 606 32.17821782 | |
| 607 32.28995058 | |
| 608 32.23684211 | |
| 609 32.18390805 | |
| 610 32.13114754 | |
| 611 32.07855974 | |
| 612 32.02614379 | |
| 613 32.13703100 | |
| 614 32.24755700 | |
| 615 32.19512195 | |
| 616 32.30519481 | |
| 617 32.25283630 | |
| 618 32.20064725 | |
| 619 32.14862682 | |
| 620 32.09677419 | |
| 621 32.04508857 | |
| 622 31.99356913 | |
| 623 31.94221509 | |
| 624 31.89102564 | |
| 625 31.84000000 | |
| 626 31.94888179 | |
| 627 31.89792663 | |
| 628 31.84713376 | |
| 629 31.95548490 | |
| 630 32.06349206 | |
| 631 32.01267829 | |
| 632 32.12025316 | |
| 633 32.06951027 | |
| 634 32.01892744 | |
| 635 31.96850394 | |
| 636 31.91823899 | |
| 637 31.86813187 | |
| 638 31.81818182 | |
| 639 31.76838811 | |
| 640 31.87500000 | |
| 641 31.82527301 | |
| 642 31.93146417 | |
| 643 31.88180404 | |
| 644 31.98757764 | |
| 645 31.93798450 | |
| 646 31.88854489 | |
| 647 31.83925811 | |
| 648 31.79012346 | |
| 649 31.74114022 | |
| 650 31.69230769 | |
| 651 31.64362519 | |
| 652 31.74846626 | |
| 653 31.69984686 | |
| 654 31.80428135 | |
| 655 31.75572519 | |
| 656 31.85975610 | |
| 657 31.81126332 | |
| 658 31.91489362 | |
| 659 31.86646434 | |
| 660 31.81818182 | |
| 661 31.77004539 | |
| 662 31.72205438 | |
| 663 31.67420814 | |
| 664 31.77710843 | |
| 665 31.72932331 | |
| 666 31.68168168 | |
| 667 31.78410795 | |
| 668 31.88622754 | |
| 669 31.83856502 | |
| 670 31.79104478 | |
| 671 31.74366617 | |
| 672 31.69642857 | |
| 673 31.64933135 | |
| 674 31.60237389 | |
| 675 31.55555556 | |
| 676 31.50887574 | |
| 677 31.46233383 | |
| 678 31.56342183 | |
| 679 31.51693667 | |
| 680 31.61764706 | |
| 681 31.57121880 | |
| 682 31.52492669 | |
| 683 31.62518302 | |
| 684 31.57894737 | |
| 685 31.53284672 | |
| 686 31.48688047 | |
| 687 31.44104803 | |
| 688 31.39534884 | |
| 689 31.34978229 | |
| 690 31.30434783 | |
| 691 31.40376266 | |
| 692 31.35838150 | |
| 693 31.31313131 | |
| 694 31.41210375 | |
| 695 31.51079137 | |
| 696 31.60919540 | |
| 697 31.56384505 | |
| 698 31.51862464 | |
| 699 31.61659514 | |
| 700 31.57142857 | |
| 701 31.66904422 | |
| 702 31.76638177 | |
| 703 31.86344239 | |
| 704 31.81818182 | |
| 705 31.77304965 | |
| 706 31.72804533 | |
| 707 31.68316832 | |
| 708 31.63841808 | |
| 709 31.59379408 | |
| 710 31.69014085 | |
| 711 31.78621660 | |
| 712 31.74157303 | |
| 713 31.83730715 | |
| 714 31.93277311 | |
| 715 31.88811189 | |
| 716 31.98324022 | |
| 717 31.93863319 | |
| 718 31.89415042 | |
| 719 31.84979138 | |
| 720 31.80555556 | |
| 721 31.76144244 | |
| 722 31.71745152 | |
| 723 31.81189488 | |
| 724 31.76795580 | |
| 725 31.86206897 | |
| 726 31.95592287 | |
| 727 31.91196699 | |
| 728 31.86813187 | |
| 729 31.82441701 | |
| 730 31.78082192 | |
| 731 31.73734610 | |
| 732 31.83060109 | |
| 733 31.92360164 | |
| 734 32.01634877 | |
| 735 31.97278912 | |
| 736 31.92934783 | |
| 737 32.02170963 | |
| 738 31.97831978 | |
| 739 32.07036536 | |
| 740 32.16216216 | |
| 741 32.11875843 | |
| 742 32.21024259 | |
| 743 32.30148048 | |
| 744 32.39247312 | |
| 745 32.34899329 | |
| 746 32.30563003 | |
| 747 32.26238286 | |
| 748 32.35294118 | |
| 749 32.30974633 | |
| 750 32.40000000 | |
| Final result: 32.4000 ±1.7100 | |
| Random chance: 25.0000 ±1.5822 | |