| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 869 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 869 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 66.66666667 | |
| 4 50.00000000 | |
| 5 60.00000000 | |
| 6 66.66666667 | |
| 7 57.14285714 | |
| 8 62.50000000 | |
| 9 55.55555556 | |
| 10 50.00000000 | |
| 11 45.45454545 | |
| 12 50.00000000 | |
| 13 46.15384615 | |
| 14 50.00000000 | |
| 15 53.33333333 | |
| 16 50.00000000 | |
| 17 52.94117647 | |
| 18 50.00000000 | |
| 19 47.36842105 | |
| 20 45.00000000 | |
| 21 42.85714286 | |
| 22 45.45454545 | |
| 23 43.47826087 | |
| 24 45.83333333 | |
| 25 44.00000000 | |
| 26 42.30769231 | |
| 27 44.44444444 | |
| 28 42.85714286 | |
| 29 41.37931034 | |
| 30 43.33333333 | |
| 31 41.93548387 | |
| 32 43.75000000 | |
| 33 45.45454545 | |
| 34 47.05882353 | |
| 35 45.71428571 | |
| 36 44.44444444 | |
| 37 45.94594595 | |
| 38 44.73684211 | |
| 39 43.58974359 | |
| 40 45.00000000 | |
| 41 46.34146341 | |
| 42 45.23809524 | |
| 43 46.51162791 | |
| 44 45.45454545 | |
| 45 44.44444444 | |
| 46 45.65217391 | |
| 47 46.80851064 | |
| 48 47.91666667 | |
| 49 46.93877551 | |
| 50 48.00000000 | |
| 51 47.05882353 | |
| 52 48.07692308 | |
| 53 49.05660377 | |
| 54 48.14814815 | |
| 55 47.27272727 | |
| 56 46.42857143 | |
| 57 47.36842105 | |
| 58 48.27586207 | |
| 59 47.45762712 | |
| 60 48.33333333 | |
| 61 49.18032787 | |
| 62 48.38709677 | |
| 63 49.20634921 | |
| 64 50.00000000 | |
| 65 49.23076923 | |
| 66 50.00000000 | |
| 67 50.74626866 | |
| 68 51.47058824 | |
| 69 52.17391304 | |
| 70 51.42857143 | |
| 71 50.70422535 | |
| 72 50.00000000 | |
| 73 50.68493151 | |
| 74 51.35135135 | |
| 75 52.00000000 | |
| 76 52.63157895 | |
| 77 53.24675325 | |
| 78 53.84615385 | |
| 79 53.16455696 | |
| 80 52.50000000 | |
| 81 51.85185185 | |
| 82 52.43902439 | |
| 83 51.80722892 | |
| 84 52.38095238 | |
| 85 52.94117647 | |
| 86 53.48837209 | |
| 87 54.02298851 | |
| 88 54.54545455 | |
| 89 55.05617978 | |
| 90 55.55555556 | |
| 91 56.04395604 | |
| 92 56.52173913 | |
| 93 55.91397849 | |
| 94 55.31914894 | |
| 95 55.78947368 | |
| 96 56.25000000 | |
| 97 55.67010309 | |
| 98 56.12244898 | |
| 99 56.56565657 | |
| 100 56.00000000 | |
| 101 55.44554455 | |
| 102 55.88235294 | |
| 103 55.33980583 | |
| 104 54.80769231 | |
| 105 55.23809524 | |
| 106 54.71698113 | |
| 107 55.14018692 | |
| 108 54.62962963 | |
| 109 54.12844037 | |
| 110 54.54545455 | |
| 111 54.95495495 | |
| 112 55.35714286 | |
| 113 54.86725664 | |
| 114 54.38596491 | |
| 115 54.78260870 | |
| 116 54.31034483 | |
| 117 54.70085470 | |
| 118 55.08474576 | |
| 119 54.62184874 | |
| 120 55.00000000 | |
| 121 54.54545455 | |
| 122 54.91803279 | |
| 123 55.28455285 | |
| 124 54.83870968 | |
| 125 55.20000000 | |
| 126 55.55555556 | |
| 127 55.90551181 | |
| 128 55.46875000 | |
| 129 55.03875969 | |
| 130 54.61538462 | |
| 131 54.96183206 | |
| 132 55.30303030 | |
| 133 55.63909774 | |
| 134 55.22388060 | |
| 135 54.81481481 | |
| 136 54.41176471 | |
| 137 54.74452555 | |
| 138 55.07246377 | |
| 139 54.67625899 | |
| 140 55.00000000 | |
| 141 55.31914894 | |
| 142 55.63380282 | |
| 143 55.94405594 | |
| 144 55.55555556 | |
| 145 55.86206897 | |
| 146 55.47945205 | |
| 147 55.78231293 | |
| 148 55.40540541 | |
| 149 55.70469799 | |
| 150 56.00000000 | |
| 151 55.62913907 | |
| 152 55.92105263 | |
| 153 56.20915033 | |
| 154 55.84415584 | |
| 155 55.48387097 | |
| 156 55.76923077 | |
| 157 56.05095541 | |
| 158 56.32911392 | |
| 159 56.60377358 | |
| 160 56.25000000 | |
| 161 55.90062112 | |
| 162 56.17283951 | |
| 163 55.82822086 | |
| 164 55.48780488 | |
| 165 55.75757576 | |
| 166 56.02409639 | |
| 167 55.68862275 | |
| 168 55.95238095 | |
| 169 55.62130178 | |
| 170 55.88235294 | |
| 171 56.14035088 | |
| 172 55.81395349 | |
| 173 56.06936416 | |
| 174 56.32183908 | |
| 175 56.00000000 | |
| 176 55.68181818 | |
| 177 55.93220339 | |
| 178 55.61797753 | |
| 179 55.30726257 | |
| 180 55.00000000 | |
| 181 54.69613260 | |
| 182 54.94505495 | |
| 183 54.64480874 | |
| 184 54.89130435 | |
| 185 55.13513514 | |
| 186 55.37634409 | |
| 187 55.08021390 | |
| 188 54.78723404 | |
| 189 54.49735450 | |
| 190 54.21052632 | |
| 191 54.45026178 | |
| 192 54.16666667 | |
| 193 53.88601036 | |
| 194 54.12371134 | |
| 195 54.35897436 | |
| 196 54.59183673 | |
| 197 54.82233503 | |
| 198 55.05050505 | |
| 199 55.27638191 | |
| 200 55.50000000 | |
| 201 55.72139303 | |
| 202 55.94059406 | |
| 203 55.66502463 | |
| 204 55.88235294 | |
| 205 55.60975610 | |
| 206 55.33980583 | |
| 207 55.55555556 | |
| 208 55.76923077 | |
| 209 55.98086124 | |
| 210 56.19047619 | |
| 211 56.39810427 | |
| 212 56.13207547 | |
| 213 55.86854460 | |
| 214 56.07476636 | |
| 215 56.27906977 | |
| 216 56.01851852 | |
| 217 55.76036866 | |
| 218 55.96330275 | |
| 219 56.16438356 | |
| 220 55.90909091 | |
| 221 55.65610860 | |
| 222 55.40540541 | |
| 223 55.15695067 | |
| 224 55.35714286 | |
| 225 55.11111111 | |
| 226 54.86725664 | |
| 227 55.06607930 | |
| 228 55.26315789 | |
| 229 55.45851528 | |
| 230 55.65217391 | |
| 231 55.41125541 | |
| 232 55.17241379 | |
| 233 55.36480687 | |
| 234 55.12820513 | |
| 235 54.89361702 | |
| 236 55.08474576 | |
| 237 54.85232068 | |
| 238 55.04201681 | |
| 239 55.23012552 | |
| 240 55.41666667 | |
| 241 55.60165975 | |
| 242 55.37190083 | |
| 243 55.14403292 | |
| 244 54.91803279 | |
| 245 54.69387755 | |
| 246 54.47154472 | |
| 247 54.65587045 | |
| 248 54.43548387 | |
| 249 54.21686747 | |
| 250 54.40000000 | |
| 251 54.58167331 | |
| 252 54.76190476 | |
| 253 54.54545455 | |
| 254 54.33070866 | |
| 255 54.50980392 | |
| 256 54.68750000 | |
| 257 54.86381323 | |
| 258 55.03875969 | |
| 259 54.82625483 | |
| 260 55.00000000 | |
| 261 55.17241379 | |
| 262 55.34351145 | |
| 263 55.51330798 | |
| 264 55.68181818 | |
| 265 55.84905660 | |
| 266 55.63909774 | |
| 267 55.80524345 | |
| 268 55.59701493 | |
| 269 55.76208178 | |
| 270 55.92592593 | |
| 271 55.71955720 | |
| 272 55.51470588 | |
| 273 55.67765568 | |
| 274 55.47445255 | |
| 275 55.27272727 | |
| 276 55.43478261 | |
| 277 55.59566787 | |
| 278 55.39568345 | |
| 279 55.19713262 | |
| 280 55.00000000 | |
| 281 55.16014235 | |
| 282 55.31914894 | |
| 283 55.12367491 | |
| 284 55.28169014 | |
| 285 55.08771930 | |
| 286 55.24475524 | |
| 287 55.05226481 | |
| 288 55.20833333 | |
| 289 55.01730104 | |
| 290 55.17241379 | |
| 291 55.32646048 | |
| 292 55.13698630 | |
| 293 55.29010239 | |
| 294 55.10204082 | |
| 295 55.25423729 | |
| 296 55.40540541 | |
| 297 55.55555556 | |
| 298 55.36912752 | |
| 299 55.51839465 | |
| 300 55.33333333 | |
| 301 55.14950166 | |
| 302 55.29801325 | |
| 303 55.11551155 | |
| 304 55.26315789 | |
| 305 55.40983607 | |
| 306 55.22875817 | |
| 307 55.37459283 | |
| 308 55.51948052 | |
| 309 55.66343042 | |
| 310 55.80645161 | |
| 311 55.94855305 | |
| 312 55.76923077 | |
| 313 55.91054313 | |
| 314 56.05095541 | |
| 315 55.87301587 | |
| 316 56.01265823 | |
| 317 56.15141956 | |
| 318 55.97484277 | |
| 319 55.79937304 | |
| 320 55.62500000 | |
| 321 55.45171340 | |
| 322 55.59006211 | |
| 323 55.41795666 | |
| 324 55.55555556 | |
| 325 55.38461538 | |
| 326 55.52147239 | |
| 327 55.35168196 | |
| 328 55.18292683 | |
| 329 55.01519757 | |
| 330 55.15151515 | |
| 331 55.28700906 | |
| 332 55.12048193 | |
| 333 55.25525526 | |
| 334 55.08982036 | |
| 335 54.92537313 | |
| 336 54.76190476 | |
| 337 54.59940653 | |
| 338 54.43786982 | |
| 339 54.57227139 | |
| 340 54.70588235 | |
| 341 54.83870968 | |
| 342 54.67836257 | |
| 343 54.81049563 | |
| 344 54.94186047 | |
| 345 54.78260870 | |
| 346 54.91329480 | |
| 347 54.75504323 | |
| 348 54.59770115 | |
| 349 54.72779370 | |
| 350 54.57142857 | |
| 351 54.70085470 | |
| 352 54.54545455 | |
| 353 54.39093484 | |
| 354 54.23728814 | |
| 355 54.08450704 | |
| 356 53.93258427 | |
| 357 53.78151261 | |
| 358 53.91061453 | |
| 359 53.76044568 | |
| 360 53.88888889 | |
| 361 54.01662050 | |
| 362 54.14364641 | |
| 363 54.26997245 | |
| 364 54.12087912 | |
| 365 53.97260274 | |
| 366 53.82513661 | |
| 367 53.67847411 | |
| 368 53.80434783 | |
| 369 53.92953930 | |
| 370 53.78378378 | |
| 371 53.63881402 | |
| 372 53.49462366 | |
| 373 53.35120643 | |
| 374 53.47593583 | |
| 375 53.60000000 | |
| 376 53.72340426 | |
| 377 53.84615385 | |
| 378 53.96825397 | |
| 379 53.82585752 | |
| 380 53.94736842 | |
| 381 53.80577428 | |
| 382 53.66492147 | |
| 383 53.52480418 | |
| 384 53.38541667 | |
| 385 53.50649351 | |
| 386 53.62694301 | |
| 387 53.48837209 | |
| 388 53.35051546 | |
| 389 53.21336761 | |
| 390 53.33333333 | |
| 391 53.19693095 | |
| 392 53.31632653 | |
| 393 53.18066158 | |
| 394 53.29949239 | |
| 395 53.41772152 | |
| 396 53.28282828 | |
| 397 53.40050378 | |
| 398 53.26633166 | |
| 399 53.13283208 | |
| 400 53.25000000 | |
| 401 53.11720698 | |
| 402 53.23383085 | |
| 403 53.34987593 | |
| 404 53.46534653 | |
| 405 53.58024691 | |
| 406 53.69458128 | |
| 407 53.80835381 | |
| 408 53.92156863 | |
| 409 53.78973105 | |
| 410 53.90243902 | |
| 411 54.01459854 | |
| 412 54.12621359 | |
| 413 53.99515738 | |
| 414 54.10628019 | |
| 415 53.97590361 | |
| 416 53.84615385 | |
| 417 53.71702638 | |
| 418 53.58851675 | |
| 419 53.46062053 | |
| 420 53.57142857 | |
| 421 53.44418052 | |
| 422 53.55450237 | |
| 423 53.66430260 | |
| 424 53.53773585 | |
| 425 53.41176471 | |
| 426 53.28638498 | |
| 427 53.16159251 | |
| 428 53.03738318 | |
| 429 53.14685315 | |
| 430 53.25581395 | |
| 431 53.36426914 | |
| 432 53.24074074 | |
| 433 53.11778291 | |
| 434 52.99539171 | |
| 435 53.10344828 | |
| 436 53.21100917 | |
| 437 53.31807780 | |
| 438 53.19634703 | |
| 439 53.30296128 | |
| 440 53.40909091 | |
| 441 53.28798186 | |
| 442 53.16742081 | |
| 443 53.04740406 | |
| 444 52.92792793 | |
| 445 53.03370787 | |
| 446 53.13901345 | |
| 447 53.24384787 | |
| 448 53.34821429 | |
| 449 53.45211581 | |
| 450 53.33333333 | |
| 451 53.43680710 | |
| 452 53.53982301 | |
| 453 53.64238411 | |
| 454 53.52422907 | |
| 455 53.40659341 | |
| 456 53.28947368 | |
| 457 53.17286652 | |
| 458 53.05676856 | |
| 459 53.15904139 | |
| 460 53.04347826 | |
| 461 52.92841649 | |
| 462 53.03030303 | |
| 463 52.91576674 | |
| 464 52.80172414 | |
| 465 52.90322581 | |
| 466 53.00429185 | |
| 467 52.89079229 | |
| 468 52.99145299 | |
| 469 52.87846482 | |
| 470 52.97872340 | |
| 471 53.07855626 | |
| 472 53.17796610 | |
| 473 53.06553911 | |
| 474 52.95358650 | |
| 475 52.84210526 | |
| 476 52.73109244 | |
| 477 52.83018868 | |
| 478 52.92887029 | |
| 479 53.02713987 | |
| 480 52.91666667 | |
| 481 52.80665281 | |
| 482 52.90456432 | |
| 483 53.00207039 | |
| 484 53.09917355 | |
| 485 53.19587629 | |
| 486 53.29218107 | |
| 487 53.38809035 | |
| 488 53.48360656 | |
| 489 53.37423313 | |
| 490 53.26530612 | |
| 491 53.36048880 | |
| 492 53.25203252 | |
| 493 53.14401623 | |
| 494 53.23886640 | |
| 495 53.13131313 | |
| 496 53.02419355 | |
| 497 53.11871227 | |
| 498 53.21285141 | |
| 499 53.30661323 | |
| 500 53.40000000 | |
| 501 53.49301397 | |
| 502 53.58565737 | |
| 503 53.67793241 | |
| 504 53.76984127 | |
| 505 53.86138614 | |
| 506 53.75494071 | |
| 507 53.64891519 | |
| 508 53.54330709 | |
| 509 53.63457760 | |
| 510 53.72549020 | |
| 511 53.62035225 | |
| 512 53.71093750 | |
| 513 53.60623782 | |
| 514 53.69649805 | |
| 515 53.78640777 | |
| 516 53.68217054 | |
| 517 53.57833656 | |
| 518 53.47490347 | |
| 519 53.37186898 | |
| 520 53.26923077 | |
| 521 53.35892514 | |
| 522 53.44827586 | |
| 523 53.34608031 | |
| 524 53.24427481 | |
| 525 53.14285714 | |
| 526 53.04182510 | |
| 527 52.94117647 | |
| 528 52.84090909 | |
| 529 52.74102079 | |
| 530 52.83018868 | |
| 531 52.91902072 | |
| 532 53.00751880 | |
| 533 53.09568480 | |
| 534 52.99625468 | |
| 535 52.89719626 | |
| 536 52.79850746 | |
| 537 52.88640596 | |
| 538 52.78810409 | |
| 539 52.87569573 | |
| 540 52.77777778 | |
| 541 52.68022181 | |
| 542 52.76752768 | |
| 543 52.85451197 | |
| 544 52.75735294 | |
| 545 52.84403670 | |
| 546 52.74725275 | |
| 547 52.65082267 | |
| 548 52.73722628 | |
| 549 52.82331512 | |
| 550 52.72727273 | |
| 551 52.63157895 | |
| 552 52.53623188 | |
| 553 52.62206148 | |
| 554 52.52707581 | |
| 555 52.61261261 | |
| 556 52.51798561 | |
| 557 52.60323160 | |
| 558 52.50896057 | |
| 559 52.59391771 | |
| 560 52.67857143 | |
| 561 52.76292335 | |
| 562 52.84697509 | |
| 563 52.93072824 | |
| 564 52.83687943 | |
| 565 52.74336283 | |
| 566 52.82685512 | |
| 567 52.91005291 | |
| 568 52.99295775 | |
| 569 52.89982425 | |
| 570 52.80701754 | |
| 571 52.88966725 | |
| 572 52.79720280 | |
| 573 52.70506108 | |
| 574 52.78745645 | |
| 575 52.86956522 | |
| 576 52.95138889 | |
| 577 53.03292894 | |
| 578 53.11418685 | |
| 579 53.19516408 | |
| 580 53.27586207 | |
| 581 53.18416523 | |
| 582 53.09278351 | |
| 583 53.17324185 | |
| 584 53.25342466 | |
| 585 53.33333333 | |
| 586 53.41296928 | |
| 587 53.32197615 | |
| 588 53.23129252 | |
| 589 53.31069610 | |
| 590 53.38983051 | |
| 591 53.46869712 | |
| 592 53.54729730 | |
| 593 53.45699831 | |
| 594 53.53535354 | |
| 595 53.61344538 | |
| 596 53.52348993 | |
| 597 53.43383585 | |
| 598 53.34448161 | |
| 599 53.42237062 | |
| 600 53.33333333 | |
| 601 53.41098170 | |
| 602 53.32225914 | |
| 603 53.23383085 | |
| 604 53.14569536 | |
| 605 53.05785124 | |
| 606 52.97029703 | |
| 607 52.88303130 | |
| 608 52.96052632 | |
| 609 52.87356322 | |
| 610 52.95081967 | |
| 611 52.86415712 | |
| 612 52.77777778 | |
| 613 52.69168026 | |
| 614 52.60586319 | |
| 615 52.68292683 | |
| 616 52.75974026 | |
| 617 52.83630470 | |
| 618 52.75080906 | |
| 619 52.66558966 | |
| 620 52.74193548 | |
| 621 52.65700483 | |
| 622 52.57234727 | |
| 623 52.64847512 | |
| 624 52.56410256 | |
| 625 52.48000000 | |
| 626 52.39616613 | |
| 627 52.47208931 | |
| 628 52.54777070 | |
| 629 52.62321145 | |
| 630 52.53968254 | |
| 631 52.45641838 | |
| 632 52.37341772 | |
| 633 52.29067930 | |
| 634 52.20820189 | |
| 635 52.12598425 | |
| 636 52.20125786 | |
| 637 52.11930926 | |
| 638 52.19435737 | |
| 639 52.11267606 | |
| 640 52.03125000 | |
| 641 52.10608424 | |
| 642 52.02492212 | |
| 643 51.94401244 | |
| 644 52.01863354 | |
| 645 51.93798450 | |
| 646 51.85758514 | |
| 647 51.77743431 | |
| 648 51.69753086 | |
| 649 51.61787365 | |
| 650 51.53846154 | |
| 651 51.61290323 | |
| 652 51.53374233 | |
| 653 51.45482389 | |
| 654 51.52905199 | |
| 655 51.45038168 | |
| 656 51.52439024 | |
| 657 51.44596651 | |
| 658 51.51975684 | |
| 659 51.44157815 | |
| 660 51.51515152 | |
| 661 51.43721634 | |
| 662 51.51057402 | |
| 663 51.58371041 | |
| 664 51.65662651 | |
| 665 51.72932331 | |
| 666 51.80180180 | |
| 667 51.87406297 | |
| 668 51.79640719 | |
| 669 51.71898356 | |
| 670 51.79104478 | |
| 671 51.71385991 | |
| 672 51.63690476 | |
| 673 51.56017831 | |
| 674 51.48367953 | |
| 675 51.40740741 | |
| 676 51.33136095 | |
| 677 51.40324963 | |
| 678 51.47492625 | |
| 679 51.54639175 | |
| 680 51.61764706 | |
| 681 51.54185022 | |
| 682 51.46627566 | |
| 683 51.39092240 | |
| 684 51.31578947 | |
| 685 51.38686131 | |
| 686 51.31195335 | |
| 687 51.23726346 | |
| 688 51.30813953 | |
| 689 51.37880987 | |
| 690 51.44927536 | |
| 691 51.51953690 | |
| 692 51.58959538 | |
| 693 51.65945166 | |
| 694 51.72910663 | |
| 695 51.79856115 | |
| 696 51.72413793 | |
| 697 51.79340029 | |
| 698 51.71919771 | |
| 699 51.64520744 | |
| 700 51.71428571 | |
| 701 51.78316690 | |
| 702 51.85185185 | |
| 703 51.77809388 | |
| 704 51.84659091 | |
| 705 51.91489362 | |
| 706 51.84135977 | |
| 707 51.90947666 | |
| 708 51.97740113 | |
| 709 52.04513399 | |
| 710 52.11267606 | |
| 711 52.18002813 | |
| 712 52.24719101 | |
| 713 52.31416550 | |
| 714 52.24089636 | |
| 715 52.16783217 | |
| 716 52.23463687 | |
| 717 52.16178522 | |
| 718 52.08913649 | |
| 719 52.01668985 | |
| 720 51.94444444 | |
| 721 51.87239945 | |
| 722 51.80055402 | |
| 723 51.86721992 | |
| 724 51.93370166 | |
| 725 52.00000000 | |
| 726 51.92837466 | |
| 727 51.85694635 | |
| 728 51.92307692 | |
| 729 51.98902606 | |
| 730 52.05479452 | |
| 731 52.12038304 | |
| 732 52.18579235 | |
| 733 52.25102319 | |
| 734 52.31607629 | |
| 735 52.24489796 | |
| 736 52.30978261 | |
| 737 52.23880597 | |
| 738 52.16802168 | |
| 739 52.23274696 | |
| 740 52.16216216 | |
| 741 52.22672065 | |
| 742 52.29110512 | |
| 743 52.35531629 | |
| 744 52.28494624 | |
| 745 52.34899329 | |
| 746 52.27882038 | |
| 747 52.20883534 | |
| 748 52.13903743 | |
| 749 52.06942590 | |
| 750 52.13333333 | |
| Final result: 52.1333 ±1.8253 | |
| Random chance: 25.0083 ±1.5824 | |