| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 1548 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 1548 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 0.00000000 | |
| 2 0.00000000 | |
| 3 0.00000000 | |
| 4 25.00000000 | |
| 5 20.00000000 | |
| 6 16.66666667 | |
| 7 28.57142857 | |
| 8 25.00000000 | |
| 9 22.22222222 | |
| 10 20.00000000 | |
| 11 18.18181818 | |
| 12 16.66666667 | |
| 13 23.07692308 | |
| 14 21.42857143 | |
| 15 26.66666667 | |
| 16 31.25000000 | |
| 17 29.41176471 | |
| 18 27.77777778 | |
| 19 31.57894737 | |
| 20 30.00000000 | |
| 21 28.57142857 | |
| 22 27.27272727 | |
| 23 26.08695652 | |
| 24 25.00000000 | |
| 25 28.00000000 | |
| 26 26.92307692 | |
| 27 25.92592593 | |
| 28 28.57142857 | |
| 29 31.03448276 | |
| 30 33.33333333 | |
| 31 35.48387097 | |
| 32 37.50000000 | |
| 33 36.36363636 | |
| 34 35.29411765 | |
| 35 37.14285714 | |
| 36 36.11111111 | |
| 37 35.13513514 | |
| 38 36.84210526 | |
| 39 35.89743590 | |
| 40 35.00000000 | |
| 41 36.58536585 | |
| 42 35.71428571 | |
| 43 34.88372093 | |
| 44 34.09090909 | |
| 45 33.33333333 | |
| 46 32.60869565 | |
| 47 31.91489362 | |
| 48 33.33333333 | |
| 49 34.69387755 | |
| 50 36.00000000 | |
| 51 35.29411765 | |
| 52 36.53846154 | |
| 53 35.84905660 | |
| 54 35.18518519 | |
| 55 36.36363636 | |
| 56 37.50000000 | |
| 57 36.84210526 | |
| 58 36.20689655 | |
| 59 35.59322034 | |
| 60 35.00000000 | |
| 61 34.42622951 | |
| 62 33.87096774 | |
| 63 33.33333333 | |
| 64 32.81250000 | |
| 65 32.30769231 | |
| 66 33.33333333 | |
| 67 32.83582090 | |
| 68 32.35294118 | |
| 69 31.88405797 | |
| 70 32.85714286 | |
| 71 33.80281690 | |
| 72 34.72222222 | |
| 73 34.24657534 | |
| 74 35.13513514 | |
| 75 36.00000000 | |
| 76 36.84210526 | |
| 77 37.66233766 | |
| 78 38.46153846 | |
| 79 37.97468354 | |
| 80 38.75000000 | |
| 81 38.27160494 | |
| 82 37.80487805 | |
| 83 38.55421687 | |
| 84 39.28571429 | |
| 85 38.82352941 | |
| 86 38.37209302 | |
| 87 39.08045977 | |
| 88 39.77272727 | |
| 89 39.32584270 | |
| 90 38.88888889 | |
| 91 38.46153846 | |
| 92 38.04347826 | |
| 93 37.63440860 | |
| 94 37.23404255 | |
| 95 36.84210526 | |
| 96 37.50000000 | |
| 97 37.11340206 | |
| 98 36.73469388 | |
| 99 36.36363636 | |
| 100 37.00000000 | |
| 101 36.63366337 | |
| 102 36.27450980 | |
| 103 35.92233010 | |
| 104 35.57692308 | |
| 105 35.23809524 | |
| 106 34.90566038 | |
| 107 34.57943925 | |
| 108 34.25925926 | |
| 109 33.94495413 | |
| 110 33.63636364 | |
| 111 33.33333333 | |
| 112 33.03571429 | |
| 113 33.62831858 | |
| 114 34.21052632 | |
| 115 34.78260870 | |
| 116 34.48275862 | |
| 117 34.18803419 | |
| 118 33.89830508 | |
| 119 33.61344538 | |
| 120 33.33333333 | |
| 121 33.05785124 | |
| 122 32.78688525 | |
| 123 32.52032520 | |
| 124 32.25806452 | |
| 125 32.80000000 | |
| 126 32.53968254 | |
| 127 32.28346457 | |
| 128 32.03125000 | |
| 129 31.78294574 | |
| 130 31.53846154 | |
| 131 32.06106870 | |
| 132 31.81818182 | |
| 133 31.57894737 | |
| 134 31.34328358 | |
| 135 31.11111111 | |
| 136 31.61764706 | |
| 137 31.38686131 | |
| 138 31.15942029 | |
| 139 31.65467626 | |
| 140 31.42857143 | |
| 141 31.20567376 | |
| 142 31.69014085 | |
| 143 32.16783217 | |
| 144 32.63888889 | |
| 145 32.41379310 | |
| 146 32.19178082 | |
| 147 31.97278912 | |
| 148 32.43243243 | |
| 149 32.21476510 | |
| 150 32.00000000 | |
| 151 31.78807947 | |
| 152 32.23684211 | |
| 153 32.67973856 | |
| 154 32.46753247 | |
| 155 32.25806452 | |
| 156 32.05128205 | |
| 157 32.48407643 | |
| 158 32.91139241 | |
| 159 32.70440252 | |
| 160 33.12500000 | |
| 161 32.91925466 | |
| 162 32.71604938 | |
| 163 32.51533742 | |
| 164 32.31707317 | |
| 165 32.72727273 | |
| 166 32.53012048 | |
| 167 32.93413174 | |
| 168 32.73809524 | |
| 169 32.54437870 | |
| 170 32.35294118 | |
| 171 32.16374269 | |
| 172 31.97674419 | |
| 173 31.79190751 | |
| 174 31.60919540 | |
| 175 31.42857143 | |
| 176 31.25000000 | |
| 177 31.07344633 | |
| 178 30.89887640 | |
| 179 31.28491620 | |
| 180 31.11111111 | |
| 181 30.93922652 | |
| 182 30.76923077 | |
| 183 31.14754098 | |
| 184 30.97826087 | |
| 185 31.35135135 | |
| 186 31.72043011 | |
| 187 31.55080214 | |
| 188 31.38297872 | |
| 189 31.21693122 | |
| 190 31.05263158 | |
| 191 31.41361257 | |
| 192 31.25000000 | |
| 193 31.08808290 | |
| 194 31.44329897 | |
| 195 31.79487179 | |
| 196 31.63265306 | |
| 197 31.47208122 | |
| 198 31.31313131 | |
| 199 31.15577889 | |
| 200 31.00000000 | |
| 201 31.34328358 | |
| 202 31.18811881 | |
| 203 31.03448276 | |
| 204 30.88235294 | |
| 205 31.21951220 | |
| 206 31.06796117 | |
| 207 30.91787440 | |
| 208 30.76923077 | |
| 209 30.62200957 | |
| 210 30.47619048 | |
| 211 30.80568720 | |
| 212 30.66037736 | |
| 213 30.51643192 | |
| 214 30.37383178 | |
| 215 30.23255814 | |
| 216 30.09259259 | |
| 217 29.95391705 | |
| 218 29.81651376 | |
| 219 29.68036530 | |
| 220 29.54545455 | |
| 221 29.41176471 | |
| 222 29.72972973 | |
| 223 29.59641256 | |
| 224 29.46428571 | |
| 225 29.77777778 | |
| 226 29.64601770 | |
| 227 29.95594714 | |
| 228 30.26315789 | |
| 229 30.13100437 | |
| 230 30.00000000 | |
| 231 29.87012987 | |
| 232 29.74137931 | |
| 233 29.61373391 | |
| 234 29.91452991 | |
| 235 29.78723404 | |
| 236 29.66101695 | |
| 237 29.53586498 | |
| 238 29.41176471 | |
| 239 29.28870293 | |
| 240 29.58333333 | |
| 241 29.46058091 | |
| 242 29.75206612 | |
| 243 30.04115226 | |
| 244 29.91803279 | |
| 245 30.20408163 | |
| 246 30.08130081 | |
| 247 29.95951417 | |
| 248 30.24193548 | |
| 249 30.12048193 | |
| 250 30.00000000 | |
| 251 29.88047809 | |
| 252 29.76190476 | |
| 253 29.64426877 | |
| 254 29.52755906 | |
| 255 29.80392157 | |
| 256 30.07812500 | |
| 257 30.35019455 | |
| 258 30.23255814 | |
| 259 30.50193050 | |
| 260 30.38461538 | |
| 261 30.26819923 | |
| 262 30.15267176 | |
| 263 30.03802281 | |
| 264 29.92424242 | |
| 265 29.81132075 | |
| 266 29.69924812 | |
| 267 29.58801498 | |
| 268 29.47761194 | |
| 269 29.36802974 | |
| 270 29.25925926 | |
| 271 29.15129151 | |
| 272 29.41176471 | |
| 273 29.30402930 | |
| 274 29.19708029 | |
| 275 29.09090909 | |
| 276 29.34782609 | |
| 277 29.60288809 | |
| 278 29.85611511 | |
| 279 29.74910394 | |
| 280 30.00000000 | |
| 281 29.89323843 | |
| 282 29.78723404 | |
| 283 29.68197880 | |
| 284 29.57746479 | |
| 285 29.47368421 | |
| 286 29.37062937 | |
| 287 29.61672474 | |
| 288 29.51388889 | |
| 289 29.41176471 | |
| 290 29.65517241 | |
| 291 29.89690722 | |
| 292 30.13698630 | |
| 293 30.37542662 | |
| 294 30.61224490 | |
| 295 30.50847458 | |
| 296 30.40540541 | |
| 297 30.30303030 | |
| 298 30.53691275 | |
| 299 30.43478261 | |
| 300 30.66666667 | |
| 301 30.89700997 | |
| 302 31.12582781 | |
| 303 31.02310231 | |
| 304 31.25000000 | |
| 305 31.47540984 | |
| 306 31.37254902 | |
| 307 31.59609121 | |
| 308 31.49350649 | |
| 309 31.71521036 | |
| 310 31.61290323 | |
| 311 31.83279743 | |
| 312 32.05128205 | |
| 313 32.26837061 | |
| 314 32.16560510 | |
| 315 32.06349206 | |
| 316 31.96202532 | |
| 317 31.86119874 | |
| 318 31.76100629 | |
| 319 31.97492163 | |
| 320 31.87500000 | |
| 321 31.77570093 | |
| 322 31.67701863 | |
| 323 31.57894737 | |
| 324 31.48148148 | |
| 325 31.69230769 | |
| 326 31.59509202 | |
| 327 31.80428135 | |
| 328 31.70731707 | |
| 329 31.61094225 | |
| 330 31.51515152 | |
| 331 31.41993958 | |
| 332 31.62650602 | |
| 333 31.53153153 | |
| 334 31.43712575 | |
| 335 31.64179104 | |
| 336 31.54761905 | |
| 337 31.45400593 | |
| 338 31.36094675 | |
| 339 31.26843658 | |
| 340 31.17647059 | |
| 341 31.37829912 | |
| 342 31.28654971 | |
| 343 31.19533528 | |
| 344 31.10465116 | |
| 345 31.01449275 | |
| 346 31.21387283 | |
| 347 31.41210375 | |
| 348 31.32183908 | |
| 349 31.23209169 | |
| 350 31.42857143 | |
| 351 31.33903134 | |
| 352 31.25000000 | |
| 353 31.44475921 | |
| 354 31.35593220 | |
| 355 31.26760563 | |
| 356 31.46067416 | |
| 357 31.37254902 | |
| 358 31.28491620 | |
| 359 31.19777159 | |
| 360 31.38888889 | |
| 361 31.57894737 | |
| 362 31.76795580 | |
| 363 31.95592287 | |
| 364 31.86813187 | |
| 365 31.78082192 | |
| 366 31.69398907 | |
| 367 31.60762943 | |
| 368 31.52173913 | |
| 369 31.43631436 | |
| 370 31.62162162 | |
| 371 31.53638814 | |
| 372 31.45161290 | |
| 373 31.36729223 | |
| 374 31.55080214 | |
| 375 31.73333333 | |
| 376 31.64893617 | |
| 377 31.56498674 | |
| 378 31.48148148 | |
| 379 31.39841689 | |
| 380 31.31578947 | |
| 381 31.49606299 | |
| 382 31.41361257 | |
| 383 31.59268930 | |
| 384 31.51041667 | |
| 385 31.68831169 | |
| 386 31.60621762 | |
| 387 31.52454780 | |
| 388 31.44329897 | |
| 389 31.61953728 | |
| 390 31.79487179 | |
| 391 31.96930946 | |
| 392 32.14285714 | |
| 393 32.06106870 | |
| 394 31.97969543 | |
| 395 31.89873418 | |
| 396 32.07070707 | |
| 397 31.98992443 | |
| 398 31.90954774 | |
| 399 32.08020050 | |
| 400 32.00000000 | |
| 401 32.16957606 | |
| 402 32.08955224 | |
| 403 32.00992556 | |
| 404 31.93069307 | |
| 405 31.85185185 | |
| 406 31.77339901 | |
| 407 31.69533170 | |
| 408 31.61764706 | |
| 409 31.78484108 | |
| 410 31.70731707 | |
| 411 31.63017032 | |
| 412 31.55339806 | |
| 413 31.71912833 | |
| 414 31.88405797 | |
| 415 32.04819277 | |
| 416 31.97115385 | |
| 417 31.89448441 | |
| 418 31.81818182 | |
| 419 31.98090692 | |
| 420 32.14285714 | |
| 421 32.06650831 | |
| 422 31.99052133 | |
| 423 31.91489362 | |
| 424 32.07547170 | |
| 425 32.00000000 | |
| 426 31.92488263 | |
| 427 31.85011710 | |
| 428 31.77570093 | |
| 429 31.93473193 | |
| 430 31.86046512 | |
| 431 32.01856148 | |
| 432 32.17592593 | |
| 433 32.10161663 | |
| 434 32.25806452 | |
| 435 32.18390805 | |
| 436 32.11009174 | |
| 437 32.03661327 | |
| 438 32.19178082 | |
| 439 32.11845103 | |
| 440 32.04545455 | |
| 441 31.97278912 | |
| 442 31.90045249 | |
| 443 31.82844244 | |
| 444 31.75675676 | |
| 445 31.68539326 | |
| 446 31.61434978 | |
| 447 31.54362416 | |
| 448 31.47321429 | |
| 449 31.40311804 | |
| 450 31.33333333 | |
| 451 31.26385809 | |
| 452 31.41592920 | |
| 453 31.56732892 | |
| 454 31.49779736 | |
| 455 31.42857143 | |
| 456 31.57894737 | |
| 457 31.50984683 | |
| 458 31.44104803 | |
| 459 31.59041394 | |
| 460 31.73913043 | |
| 461 31.67028200 | |
| 462 31.60173160 | |
| 463 31.53347732 | |
| 464 31.46551724 | |
| 465 31.39784946 | |
| 466 31.33047210 | |
| 467 31.47751606 | |
| 468 31.41025641 | |
| 469 31.55650320 | |
| 470 31.70212766 | |
| 471 31.63481953 | |
| 472 31.56779661 | |
| 473 31.71247357 | |
| 474 31.64556962 | |
| 475 31.57894737 | |
| 476 31.72268908 | |
| 477 31.65618449 | |
| 478 31.79916318 | |
| 479 31.94154489 | |
| 480 31.87500000 | |
| 481 31.80873181 | |
| 482 31.74273859 | |
| 483 31.67701863 | |
| 484 31.61157025 | |
| 485 31.54639175 | |
| 486 31.48148148 | |
| 487 31.62217659 | |
| 488 31.55737705 | |
| 489 31.49284254 | |
| 490 31.63265306 | |
| 491 31.77189409 | |
| 492 31.70731707 | |
| 493 31.64300203 | |
| 494 31.78137652 | |
| 495 31.91919192 | |
| 496 32.05645161 | |
| 497 31.99195171 | |
| 498 31.92771084 | |
| 499 31.86372745 | |
| 500 32.00000000 | |
| 501 31.93612774 | |
| 502 31.87250996 | |
| 503 31.80914513 | |
| 504 31.94444444 | |
| 505 31.88118812 | |
| 506 32.01581028 | |
| 507 31.95266272 | |
| 508 31.88976378 | |
| 509 31.82711198 | |
| 510 31.96078431 | |
| 511 31.89823875 | |
| 512 31.83593750 | |
| 513 31.77387914 | |
| 514 31.71206226 | |
| 515 31.65048544 | |
| 516 31.58914729 | |
| 517 31.72147002 | |
| 518 31.85328185 | |
| 519 31.79190751 | |
| 520 31.73076923 | |
| 521 31.66986564 | |
| 522 31.80076628 | |
| 523 31.93116635 | |
| 524 32.06106870 | |
| 525 32.00000000 | |
| 526 31.93916350 | |
| 527 32.06831120 | |
| 528 32.00757576 | |
| 529 31.94706994 | |
| 530 32.07547170 | |
| 531 32.01506591 | |
| 532 31.95488722 | |
| 533 31.89493433 | |
| 534 31.83520599 | |
| 535 31.96261682 | |
| 536 31.90298507 | |
| 537 32.02979516 | |
| 538 32.15613383 | |
| 539 32.09647495 | |
| 540 32.03703704 | |
| 541 32.16266174 | |
| 542 32.28782288 | |
| 543 32.22836096 | |
| 544 32.16911765 | |
| 545 32.11009174 | |
| 546 32.05128205 | |
| 547 31.99268739 | |
| 548 32.11678832 | |
| 549 32.05828780 | |
| 550 32.00000000 | |
| 551 31.94192377 | |
| 552 31.88405797 | |
| 553 31.82640145 | |
| 554 31.76895307 | |
| 555 31.71171171 | |
| 556 31.65467626 | |
| 557 31.59784560 | |
| 558 31.72043011 | |
| 559 31.66368515 | |
| 560 31.60714286 | |
| 561 31.55080214 | |
| 562 31.49466192 | |
| 563 31.61634103 | |
| 564 31.56028369 | |
| 565 31.50442478 | |
| 566 31.44876325 | |
| 567 31.39329806 | |
| 568 31.51408451 | |
| 569 31.45869947 | |
| 570 31.40350877 | |
| 571 31.52364273 | |
| 572 31.46853147 | |
| 573 31.41361257 | |
| 574 31.53310105 | |
| 575 31.47826087 | |
| 576 31.59722222 | |
| 577 31.71577123 | |
| 578 31.83391003 | |
| 579 31.77892919 | |
| 580 31.72413793 | |
| 581 31.66953528 | |
| 582 31.61512027 | |
| 583 31.56089194 | |
| 584 31.67808219 | |
| 585 31.62393162 | |
| 586 31.56996587 | |
| 587 31.51618399 | |
| 588 31.46258503 | |
| 589 31.40916808 | |
| 590 31.35593220 | |
| 591 31.30287648 | |
| 592 31.25000000 | |
| 593 31.19730185 | |
| 594 31.31313131 | |
| 595 31.26050420 | |
| 596 31.37583893 | |
| 597 31.32328308 | |
| 598 31.27090301 | |
| 599 31.38564274 | |
| 600 31.33333333 | |
| 601 31.44758735 | |
| 602 31.39534884 | |
| 603 31.50912106 | |
| 604 31.45695364 | |
| 605 31.57024793 | |
| 606 31.51815182 | |
| 607 31.63097199 | |
| 608 31.57894737 | |
| 609 31.52709360 | |
| 610 31.47540984 | |
| 611 31.42389525 | |
| 612 31.37254902 | |
| 613 31.32137031 | |
| 614 31.43322476 | |
| 615 31.38211382 | |
| 616 31.49350649 | |
| 617 31.60453809 | |
| 618 31.55339806 | |
| 619 31.50242326 | |
| 620 31.45161290 | |
| 621 31.40096618 | |
| 622 31.51125402 | |
| 623 31.46067416 | |
| 624 31.41025641 | |
| 625 31.36000000 | |
| 626 31.46964856 | |
| 627 31.41945774 | |
| 628 31.36942675 | |
| 629 31.47853736 | |
| 630 31.42857143 | |
| 631 31.37876387 | |
| 632 31.48734177 | |
| 633 31.43759874 | |
| 634 31.38801262 | |
| 635 31.33858268 | |
| 636 31.28930818 | |
| 637 31.24018838 | |
| 638 31.19122257 | |
| 639 31.29890454 | |
| 640 31.40625000 | |
| 641 31.35725429 | |
| 642 31.46417445 | |
| 643 31.41524106 | |
| 644 31.52173913 | |
| 645 31.47286822 | |
| 646 31.42414861 | |
| 647 31.37557960 | |
| 648 31.32716049 | |
| 649 31.27889060 | |
| 650 31.23076923 | |
| 651 31.18279570 | |
| 652 31.28834356 | |
| 653 31.24042879 | |
| 654 31.34556575 | |
| 655 31.29770992 | |
| 656 31.40243902 | |
| 657 31.50684932 | |
| 658 31.61094225 | |
| 659 31.56297420 | |
| 660 31.51515152 | |
| 661 31.46747352 | |
| 662 31.41993958 | |
| 663 31.37254902 | |
| 664 31.32530120 | |
| 665 31.27819549 | |
| 666 31.23123123 | |
| 667 31.33433283 | |
| 668 31.43712575 | |
| 669 31.39013453 | |
| 670 31.34328358 | |
| 671 31.29657228 | |
| 672 31.25000000 | |
| 673 31.20356612 | |
| 674 31.15727003 | |
| 675 31.11111111 | |
| 676 31.06508876 | |
| 677 31.01920236 | |
| 678 30.97345133 | |
| 679 30.92783505 | |
| 680 31.02941176 | |
| 681 30.98384728 | |
| 682 30.93841642 | |
| 683 31.03953148 | |
| 684 30.99415205 | |
| 685 30.94890511 | |
| 686 30.90379009 | |
| 687 30.85880640 | |
| 688 30.81395349 | |
| 689 30.76923077 | |
| 690 30.72463768 | |
| 691 30.82489146 | |
| 692 30.78034682 | |
| 693 30.73593074 | |
| 694 30.69164265 | |
| 695 30.64748201 | |
| 696 30.74712644 | |
| 697 30.70301291 | |
| 698 30.65902579 | |
| 699 30.75822604 | |
| 700 30.71428571 | |
| 701 30.81312411 | |
| 702 30.91168091 | |
| 703 31.00995733 | |
| 704 30.96590909 | |
| 705 30.92198582 | |
| 706 30.87818697 | |
| 707 30.83451202 | |
| 708 30.79096045 | |
| 709 30.74753173 | |
| 710 30.84507042 | |
| 711 30.94233474 | |
| 712 30.89887640 | |
| 713 30.99579243 | |
| 714 31.09243697 | |
| 715 31.04895105 | |
| 716 31.00558659 | |
| 717 30.96234310 | |
| 718 30.91922006 | |
| 719 30.87621697 | |
| 720 30.83333333 | |
| 721 30.79056865 | |
| 722 30.74792244 | |
| 723 30.84370678 | |
| 724 30.80110497 | |
| 725 30.89655172 | |
| 726 30.99173554 | |
| 727 30.94910591 | |
| 728 30.90659341 | |
| 729 30.86419753 | |
| 730 30.82191781 | |
| 731 30.77975376 | |
| 732 30.87431694 | |
| 733 30.96862210 | |
| 734 30.92643052 | |
| 735 30.88435374 | |
| 736 30.84239130 | |
| 737 30.93622795 | |
| 738 30.89430894 | |
| 739 30.98782138 | |
| 740 31.08108108 | |
| 741 31.03913630 | |
| 742 31.13207547 | |
| 743 31.22476447 | |
| 744 31.31720430 | |
| 745 31.27516779 | |
| 746 31.23324397 | |
| 747 31.19143240 | |
| 748 31.14973262 | |
| 749 31.10814419 | |
| 750 31.20000000 | |
| Final result: 31.2000 ±1.6929 | |
| Random chance: 25.0000 ±1.5822 | |