| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 1548 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 1548 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 33.33333333 | |
| 4 50.00000000 | |
| 5 40.00000000 | |
| 6 33.33333333 | |
| 7 42.85714286 | |
| 8 50.00000000 | |
| 9 44.44444444 | |
| 10 50.00000000 | |
| 11 45.45454545 | |
| 12 41.66666667 | |
| 13 38.46153846 | |
| 14 35.71428571 | |
| 15 40.00000000 | |
| 16 43.75000000 | |
| 17 47.05882353 | |
| 18 44.44444444 | |
| 19 47.36842105 | |
| 20 45.00000000 | |
| 21 47.61904762 | |
| 22 45.45454545 | |
| 23 43.47826087 | |
| 24 45.83333333 | |
| 25 48.00000000 | |
| 26 46.15384615 | |
| 27 44.44444444 | |
| 28 42.85714286 | |
| 29 44.82758621 | |
| 30 46.66666667 | |
| 31 48.38709677 | |
| 32 50.00000000 | |
| 33 48.48484848 | |
| 34 50.00000000 | |
| 35 51.42857143 | |
| 36 52.77777778 | |
| 37 51.35135135 | |
| 38 52.63157895 | |
| 39 51.28205128 | |
| 40 52.50000000 | |
| 41 51.21951220 | |
| 42 50.00000000 | |
| 43 51.16279070 | |
| 44 50.00000000 | |
| 45 51.11111111 | |
| 46 50.00000000 | |
| 47 48.93617021 | |
| 48 50.00000000 | |
| 49 48.97959184 | |
| 50 48.00000000 | |
| 51 47.05882353 | |
| 52 48.07692308 | |
| 53 47.16981132 | |
| 54 46.29629630 | |
| 55 47.27272727 | |
| 56 48.21428571 | |
| 57 49.12280702 | |
| 58 50.00000000 | |
| 59 49.15254237 | |
| 60 48.33333333 | |
| 61 49.18032787 | |
| 62 48.38709677 | |
| 63 47.61904762 | |
| 64 46.87500000 | |
| 65 46.15384615 | |
| 66 45.45454545 | |
| 67 44.77611940 | |
| 68 44.11764706 | |
| 69 43.47826087 | |
| 70 44.28571429 | |
| 71 45.07042254 | |
| 72 44.44444444 | |
| 73 43.83561644 | |
| 74 44.59459459 | |
| 75 45.33333333 | |
| 76 46.05263158 | |
| 77 46.75324675 | |
| 78 47.43589744 | |
| 79 46.83544304 | |
| 80 47.50000000 | |
| 81 48.14814815 | |
| 82 47.56097561 | |
| 83 48.19277108 | |
| 84 47.61904762 | |
| 85 47.05882353 | |
| 86 46.51162791 | |
| 87 47.12643678 | |
| 88 47.72727273 | |
| 89 47.19101124 | |
| 90 46.66666667 | |
| 91 46.15384615 | |
| 92 45.65217391 | |
| 93 45.16129032 | |
| 94 44.68085106 | |
| 95 44.21052632 | |
| 96 44.79166667 | |
| 97 44.32989691 | |
| 98 43.87755102 | |
| 99 43.43434343 | |
| 100 43.00000000 | |
| 101 43.56435644 | |
| 102 43.13725490 | |
| 103 42.71844660 | |
| 104 43.26923077 | |
| 105 43.80952381 | |
| 106 44.33962264 | |
| 107 43.92523364 | |
| 108 43.51851852 | |
| 109 44.03669725 | |
| 110 44.54545455 | |
| 111 44.14414414 | |
| 112 43.75000000 | |
| 113 43.36283186 | |
| 114 43.85964912 | |
| 115 44.34782609 | |
| 116 43.96551724 | |
| 117 44.44444444 | |
| 118 44.91525424 | |
| 119 44.53781513 | |
| 120 44.16666667 | |
| 121 44.62809917 | |
| 122 44.26229508 | |
| 123 43.90243902 | |
| 124 43.54838710 | |
| 125 43.20000000 | |
| 126 42.85714286 | |
| 127 42.51968504 | |
| 128 42.18750000 | |
| 129 41.86046512 | |
| 130 41.53846154 | |
| 131 41.98473282 | |
| 132 41.66666667 | |
| 133 41.35338346 | |
| 134 41.04477612 | |
| 135 40.74074074 | |
| 136 40.44117647 | |
| 137 40.14598540 | |
| 138 39.85507246 | |
| 139 40.28776978 | |
| 140 40.71428571 | |
| 141 40.42553191 | |
| 142 40.84507042 | |
| 143 41.25874126 | |
| 144 40.97222222 | |
| 145 41.37931034 | |
| 146 41.78082192 | |
| 147 41.49659864 | |
| 148 41.89189189 | |
| 149 41.61073826 | |
| 150 41.33333333 | |
| 151 41.05960265 | |
| 152 41.44736842 | |
| 153 41.17647059 | |
| 154 40.90909091 | |
| 155 40.64516129 | |
| 156 40.38461538 | |
| 157 40.76433121 | |
| 158 41.13924051 | |
| 159 40.88050314 | |
| 160 41.25000000 | |
| 161 40.99378882 | |
| 162 40.74074074 | |
| 163 40.49079755 | |
| 164 40.24390244 | |
| 165 40.60606061 | |
| 166 40.36144578 | |
| 167 40.71856287 | |
| 168 40.47619048 | |
| 169 40.82840237 | |
| 170 40.58823529 | |
| 171 40.93567251 | |
| 172 40.69767442 | |
| 173 40.46242775 | |
| 174 40.22988506 | |
| 175 40.00000000 | |
| 176 39.77272727 | |
| 177 39.54802260 | |
| 178 39.32584270 | |
| 179 39.66480447 | |
| 180 40.00000000 | |
| 181 40.33149171 | |
| 182 40.10989011 | |
| 183 40.43715847 | |
| 184 40.21739130 | |
| 185 40.54054054 | |
| 186 40.86021505 | |
| 187 41.17647059 | |
| 188 41.48936170 | |
| 189 41.79894180 | |
| 190 41.57894737 | |
| 191 41.88481675 | |
| 192 41.66666667 | |
| 193 41.45077720 | |
| 194 41.75257732 | |
| 195 42.05128205 | |
| 196 42.34693878 | |
| 197 42.13197970 | |
| 198 42.42424242 | |
| 199 42.21105528 | |
| 200 42.00000000 | |
| 201 41.79104478 | |
| 202 41.58415842 | |
| 203 41.37931034 | |
| 204 41.17647059 | |
| 205 41.46341463 | |
| 206 41.26213592 | |
| 207 41.54589372 | |
| 208 41.34615385 | |
| 209 41.14832536 | |
| 210 40.95238095 | |
| 211 41.23222749 | |
| 212 41.03773585 | |
| 213 40.84507042 | |
| 214 40.65420561 | |
| 215 40.46511628 | |
| 216 40.74074074 | |
| 217 40.55299539 | |
| 218 40.82568807 | |
| 219 41.09589041 | |
| 220 41.36363636 | |
| 221 41.17647059 | |
| 222 41.44144144 | |
| 223 41.70403587 | |
| 224 41.51785714 | |
| 225 41.77777778 | |
| 226 41.59292035 | |
| 227 41.40969163 | |
| 228 41.66666667 | |
| 229 41.92139738 | |
| 230 41.73913043 | |
| 231 41.55844156 | |
| 232 41.81034483 | |
| 233 41.63090129 | |
| 234 41.88034188 | |
| 235 41.70212766 | |
| 236 41.52542373 | |
| 237 41.35021097 | |
| 238 41.17647059 | |
| 239 41.00418410 | |
| 240 41.25000000 | |
| 241 41.07883817 | |
| 242 41.32231405 | |
| 243 41.56378601 | |
| 244 41.39344262 | |
| 245 41.63265306 | |
| 246 41.46341463 | |
| 247 41.29554656 | |
| 248 41.53225806 | |
| 249 41.36546185 | |
| 250 41.20000000 | |
| 251 41.03585657 | |
| 252 40.87301587 | |
| 253 40.71146245 | |
| 254 40.55118110 | |
| 255 40.39215686 | |
| 256 40.62500000 | |
| 257 40.85603113 | |
| 258 40.69767442 | |
| 259 40.54054054 | |
| 260 40.38461538 | |
| 261 40.61302682 | |
| 262 40.45801527 | |
| 263 40.30418251 | |
| 264 40.53030303 | |
| 265 40.75471698 | |
| 266 40.97744361 | |
| 267 40.82397004 | |
| 268 40.67164179 | |
| 269 40.89219331 | |
| 270 40.74074074 | |
| 271 40.59040590 | |
| 272 40.44117647 | |
| 273 40.29304029 | |
| 274 40.14598540 | |
| 275 40.00000000 | |
| 276 40.21739130 | |
| 277 40.43321300 | |
| 278 40.28776978 | |
| 279 40.14336918 | |
| 280 40.35714286 | |
| 281 40.21352313 | |
| 282 40.42553191 | |
| 283 40.28268551 | |
| 284 40.14084507 | |
| 285 40.00000000 | |
| 286 39.86013986 | |
| 287 40.06968641 | |
| 288 39.93055556 | |
| 289 39.79238754 | |
| 290 40.00000000 | |
| 291 40.20618557 | |
| 292 40.41095890 | |
| 293 40.27303754 | |
| 294 40.13605442 | |
| 295 40.00000000 | |
| 296 39.86486486 | |
| 297 39.73063973 | |
| 298 39.93288591 | |
| 299 39.79933110 | |
| 300 40.00000000 | |
| 301 39.86710963 | |
| 302 39.73509934 | |
| 303 39.60396040 | |
| 304 39.47368421 | |
| 305 39.67213115 | |
| 306 39.54248366 | |
| 307 39.73941368 | |
| 308 39.61038961 | |
| 309 39.48220065 | |
| 310 39.35483871 | |
| 311 39.22829582 | |
| 312 39.42307692 | |
| 313 39.29712460 | |
| 314 39.17197452 | |
| 315 39.04761905 | |
| 316 38.92405063 | |
| 317 38.80126183 | |
| 318 38.99371069 | |
| 319 39.18495298 | |
| 320 39.06250000 | |
| 321 38.94080997 | |
| 322 39.13043478 | |
| 323 39.00928793 | |
| 324 38.88888889 | |
| 325 39.07692308 | |
| 326 38.95705521 | |
| 327 39.14373089 | |
| 328 39.02439024 | |
| 329 38.90577508 | |
| 330 38.78787879 | |
| 331 38.67069486 | |
| 332 38.55421687 | |
| 333 38.43843844 | |
| 334 38.32335329 | |
| 335 38.50746269 | |
| 336 38.69047619 | |
| 337 38.57566766 | |
| 338 38.46153846 | |
| 339 38.34808260 | |
| 340 38.23529412 | |
| 341 38.41642229 | |
| 342 38.59649123 | |
| 343 38.48396501 | |
| 344 38.66279070 | |
| 345 38.55072464 | |
| 346 38.72832370 | |
| 347 38.90489914 | |
| 348 38.79310345 | |
| 349 38.96848138 | |
| 350 39.14285714 | |
| 351 39.03133903 | |
| 352 39.20454545 | |
| 353 39.37677054 | |
| 354 39.26553672 | |
| 355 39.15492958 | |
| 356 39.04494382 | |
| 357 39.21568627 | |
| 358 39.10614525 | |
| 359 38.99721448 | |
| 360 39.16666667 | |
| 361 39.33518006 | |
| 362 39.50276243 | |
| 363 39.66942149 | |
| 364 39.56043956 | |
| 365 39.45205479 | |
| 366 39.61748634 | |
| 367 39.50953678 | |
| 368 39.40217391 | |
| 369 39.29539295 | |
| 370 39.45945946 | |
| 371 39.35309973 | |
| 372 39.24731183 | |
| 373 39.14209115 | |
| 374 39.03743316 | |
| 375 39.20000000 | |
| 376 39.09574468 | |
| 377 38.99204244 | |
| 378 39.15343915 | |
| 379 39.05013193 | |
| 380 38.94736842 | |
| 381 38.84514436 | |
| 382 39.00523560 | |
| 383 39.16449086 | |
| 384 39.06250000 | |
| 385 39.22077922 | |
| 386 39.11917098 | |
| 387 39.01808786 | |
| 388 38.91752577 | |
| 389 39.07455013 | |
| 390 39.23076923 | |
| 391 39.38618926 | |
| 392 39.54081633 | |
| 393 39.44020356 | |
| 394 39.34010152 | |
| 395 39.49367089 | |
| 396 39.64646465 | |
| 397 39.54659950 | |
| 398 39.44723618 | |
| 399 39.59899749 | |
| 400 39.50000000 | |
| 401 39.65087282 | |
| 402 39.55223881 | |
| 403 39.45409429 | |
| 404 39.35643564 | |
| 405 39.50617284 | |
| 406 39.40886700 | |
| 407 39.31203931 | |
| 408 39.21568627 | |
| 409 39.36430318 | |
| 410 39.26829268 | |
| 411 39.17274939 | |
| 412 39.07766990 | |
| 413 38.98305085 | |
| 414 39.13043478 | |
| 415 39.03614458 | |
| 416 38.94230769 | |
| 417 38.84892086 | |
| 418 38.75598086 | |
| 419 38.90214797 | |
| 420 38.80952381 | |
| 421 38.71733967 | |
| 422 38.62559242 | |
| 423 38.53427896 | |
| 424 38.67924528 | |
| 425 38.58823529 | |
| 426 38.49765258 | |
| 427 38.64168618 | |
| 428 38.55140187 | |
| 429 38.69463869 | |
| 430 38.60465116 | |
| 431 38.74709977 | |
| 432 38.65740741 | |
| 433 38.56812933 | |
| 434 38.47926267 | |
| 435 38.39080460 | |
| 436 38.30275229 | |
| 437 38.21510297 | |
| 438 38.35616438 | |
| 439 38.49658314 | |
| 440 38.40909091 | |
| 441 38.54875283 | |
| 442 38.46153846 | |
| 443 38.37471783 | |
| 444 38.28828829 | |
| 445 38.20224719 | |
| 446 38.11659193 | |
| 447 38.03131991 | |
| 448 37.94642857 | |
| 449 37.86191537 | |
| 450 37.77777778 | |
| 451 37.69401330 | |
| 452 37.61061947 | |
| 453 37.74834437 | |
| 454 37.66519824 | |
| 455 37.58241758 | |
| 456 37.71929825 | |
| 457 37.85557987 | |
| 458 37.77292576 | |
| 459 37.69063181 | |
| 460 37.82608696 | |
| 461 37.74403471 | |
| 462 37.87878788 | |
| 463 37.79697624 | |
| 464 37.93103448 | |
| 465 37.84946237 | |
| 466 37.76824034 | |
| 467 37.90149893 | |
| 468 38.03418803 | |
| 469 38.16631130 | |
| 470 38.29787234 | |
| 471 38.21656051 | |
| 472 38.13559322 | |
| 473 38.26638478 | |
| 474 38.18565401 | |
| 475 38.10526316 | |
| 476 38.23529412 | |
| 477 38.15513627 | |
| 478 38.28451883 | |
| 479 38.41336117 | |
| 480 38.33333333 | |
| 481 38.25363825 | |
| 482 38.17427386 | |
| 483 38.09523810 | |
| 484 38.01652893 | |
| 485 37.93814433 | |
| 486 38.06584362 | |
| 487 38.19301848 | |
| 488 38.11475410 | |
| 489 38.03680982 | |
| 490 38.16326531 | |
| 491 38.08553971 | |
| 492 38.00813008 | |
| 493 37.93103448 | |
| 494 38.05668016 | |
| 495 38.18181818 | |
| 496 38.30645161 | |
| 497 38.22937626 | |
| 498 38.15261044 | |
| 499 38.27655311 | |
| 500 38.40000000 | |
| 501 38.32335329 | |
| 502 38.44621514 | |
| 503 38.56858847 | |
| 504 38.49206349 | |
| 505 38.61386139 | |
| 506 38.73517787 | |
| 507 38.65877712 | |
| 508 38.58267717 | |
| 509 38.70333988 | |
| 510 38.82352941 | |
| 511 38.74755382 | |
| 512 38.67187500 | |
| 513 38.59649123 | |
| 514 38.52140078 | |
| 515 38.64077670 | |
| 516 38.75968992 | |
| 517 38.87814313 | |
| 518 38.99613900 | |
| 519 38.92100193 | |
| 520 38.84615385 | |
| 521 38.77159309 | |
| 522 38.88888889 | |
| 523 38.81453155 | |
| 524 38.93129771 | |
| 525 38.85714286 | |
| 526 38.78326996 | |
| 527 38.89943074 | |
| 528 39.01515152 | |
| 529 39.13043478 | |
| 530 39.24528302 | |
| 531 39.17137476 | |
| 532 39.09774436 | |
| 533 39.02439024 | |
| 534 39.13857678 | |
| 535 39.25233645 | |
| 536 39.17910448 | |
| 537 39.29236499 | |
| 538 39.21933086 | |
| 539 39.14656772 | |
| 540 39.07407407 | |
| 541 39.00184843 | |
| 542 39.11439114 | |
| 543 39.04235727 | |
| 544 39.15441176 | |
| 545 39.08256881 | |
| 546 39.01098901 | |
| 547 39.12248629 | |
| 548 39.23357664 | |
| 549 39.16211293 | |
| 550 39.09090909 | |
| 551 39.20145191 | |
| 552 39.13043478 | |
| 553 39.05967450 | |
| 554 38.98916968 | |
| 555 38.91891892 | |
| 556 39.02877698 | |
| 557 38.95870736 | |
| 558 39.06810036 | |
| 559 38.99821109 | |
| 560 38.92857143 | |
| 561 38.85918004 | |
| 562 38.96797153 | |
| 563 39.07637655 | |
| 564 39.00709220 | |
| 565 39.11504425 | |
| 566 39.22261484 | |
| 567 39.15343915 | |
| 568 39.08450704 | |
| 569 39.01581722 | |
| 570 38.94736842 | |
| 571 38.87915937 | |
| 572 38.81118881 | |
| 573 38.74345550 | |
| 574 38.67595819 | |
| 575 38.60869565 | |
| 576 38.71527778 | |
| 577 38.64818024 | |
| 578 38.58131488 | |
| 579 38.51468048 | |
| 580 38.62068966 | |
| 581 38.55421687 | |
| 582 38.48797251 | |
| 583 38.59348199 | |
| 584 38.69863014 | |
| 585 38.80341880 | |
| 586 38.73720137 | |
| 587 38.67120954 | |
| 588 38.60544218 | |
| 589 38.53989813 | |
| 590 38.47457627 | |
| 591 38.40947547 | |
| 592 38.34459459 | |
| 593 38.44856661 | |
| 594 38.55218855 | |
| 595 38.48739496 | |
| 596 38.59060403 | |
| 597 38.52596315 | |
| 598 38.46153846 | |
| 599 38.56427379 | |
| 600 38.50000000 | |
| 601 38.43594010 | |
| 602 38.37209302 | |
| 603 38.47429519 | |
| 604 38.41059603 | |
| 605 38.51239669 | |
| 606 38.44884488 | |
| 607 38.55024712 | |
| 608 38.48684211 | |
| 609 38.42364532 | |
| 610 38.36065574 | |
| 611 38.46153846 | |
| 612 38.39869281 | |
| 613 38.49918434 | |
| 614 38.59934853 | |
| 615 38.53658537 | |
| 616 38.47402597 | |
| 617 38.41166937 | |
| 618 38.34951456 | |
| 619 38.28756058 | |
| 620 38.22580645 | |
| 621 38.16425121 | |
| 622 38.26366559 | |
| 623 38.20224719 | |
| 624 38.14102564 | |
| 625 38.08000000 | |
| 626 38.17891374 | |
| 627 38.11802233 | |
| 628 38.05732484 | |
| 629 38.15580286 | |
| 630 38.09523810 | |
| 631 38.03486529 | |
| 632 38.13291139 | |
| 633 38.07266983 | |
| 634 38.01261830 | |
| 635 37.95275591 | |
| 636 37.89308176 | |
| 637 37.99058085 | |
| 638 37.93103448 | |
| 639 37.87167449 | |
| 640 37.96875000 | |
| 641 37.90951638 | |
| 642 38.00623053 | |
| 643 37.94712286 | |
| 644 37.88819876 | |
| 645 37.82945736 | |
| 646 37.77089783 | |
| 647 37.71251932 | |
| 648 37.80864198 | |
| 649 37.75038521 | |
| 650 37.69230769 | |
| 651 37.63440860 | |
| 652 37.73006135 | |
| 653 37.67228178 | |
| 654 37.76758410 | |
| 655 37.70992366 | |
| 656 37.80487805 | |
| 657 37.74733638 | |
| 658 37.84194529 | |
| 659 37.78452200 | |
| 660 37.72727273 | |
| 661 37.67019667 | |
| 662 37.61329305 | |
| 663 37.55656109 | |
| 664 37.50000000 | |
| 665 37.44360902 | |
| 666 37.38738739 | |
| 667 37.33133433 | |
| 668 37.42514970 | |
| 669 37.51868460 | |
| 670 37.46268657 | |
| 671 37.40685544 | |
| 672 37.35119048 | |
| 673 37.29569094 | |
| 674 37.38872404 | |
| 675 37.33333333 | |
| 676 37.27810651 | |
| 677 37.22304284 | |
| 678 37.31563422 | |
| 679 37.26067747 | |
| 680 37.20588235 | |
| 681 37.29809104 | |
| 682 37.24340176 | |
| 683 37.18887262 | |
| 684 37.13450292 | |
| 685 37.22627737 | |
| 686 37.17201166 | |
| 687 37.11790393 | |
| 688 37.06395349 | |
| 689 37.01015965 | |
| 690 36.95652174 | |
| 691 37.04775687 | |
| 692 36.99421965 | |
| 693 36.94083694 | |
| 694 36.88760807 | |
| 695 36.83453237 | |
| 696 36.92528736 | |
| 697 36.87230990 | |
| 698 36.96275072 | |
| 699 37.05293276 | |
| 700 37.00000000 | |
| 701 36.94721826 | |
| 702 37.03703704 | |
| 703 37.12660028 | |
| 704 37.21590909 | |
| 705 37.16312057 | |
| 706 37.11048159 | |
| 707 37.05799151 | |
| 708 37.00564972 | |
| 709 36.95345557 | |
| 710 36.90140845 | |
| 711 36.99015471 | |
| 712 36.93820225 | |
| 713 37.02664797 | |
| 714 37.11484594 | |
| 715 37.06293706 | |
| 716 37.15083799 | |
| 717 37.09902371 | |
| 718 37.04735376 | |
| 719 36.99582754 | |
| 720 36.94444444 | |
| 721 36.89320388 | |
| 722 36.84210526 | |
| 723 36.92946058 | |
| 724 36.87845304 | |
| 725 36.96551724 | |
| 726 36.91460055 | |
| 727 37.00137552 | |
| 728 37.08791209 | |
| 729 37.03703704 | |
| 730 36.98630137 | |
| 731 36.93570451 | |
| 732 36.88524590 | |
| 733 36.97135061 | |
| 734 36.92098093 | |
| 735 36.87074830 | |
| 736 36.82065217 | |
| 737 36.90637720 | |
| 738 36.99186992 | |
| 739 37.07713126 | |
| 740 37.02702703 | |
| 741 36.97705803 | |
| 742 37.06199461 | |
| 743 37.01211306 | |
| 744 37.09677419 | |
| 745 37.04697987 | |
| 746 36.99731903 | |
| 747 36.94779116 | |
| 748 36.89839572 | |
| 749 36.84913218 | |
| 750 36.93333333 | |
| Final result: 36.9333 +/- 1.7635 | |
| Random chance: 25.0000 +/- 1.5822 | |