| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 869 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 869 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 66.66666667 | |
| 4 50.00000000 | |
| 5 60.00000000 | |
| 6 66.66666667 | |
| 7 57.14285714 | |
| 8 62.50000000 | |
| 9 55.55555556 | |
| 10 50.00000000 | |
| 11 45.45454545 | |
| 12 50.00000000 | |
| 13 46.15384615 | |
| 14 50.00000000 | |
| 15 53.33333333 | |
| 16 50.00000000 | |
| 17 52.94117647 | |
| 18 50.00000000 | |
| 19 47.36842105 | |
| 20 45.00000000 | |
| 21 42.85714286 | |
| 22 45.45454545 | |
| 23 43.47826087 | |
| 24 45.83333333 | |
| 25 44.00000000 | |
| 26 42.30769231 | |
| 27 44.44444444 | |
| 28 42.85714286 | |
| 29 41.37931034 | |
| 30 43.33333333 | |
| 31 41.93548387 | |
| 32 40.62500000 | |
| 33 42.42424242 | |
| 34 44.11764706 | |
| 35 42.85714286 | |
| 36 41.66666667 | |
| 37 43.24324324 | |
| 38 44.73684211 | |
| 39 43.58974359 | |
| 40 45.00000000 | |
| 41 46.34146341 | |
| 42 45.23809524 | |
| 43 46.51162791 | |
| 44 45.45454545 | |
| 45 44.44444444 | |
| 46 45.65217391 | |
| 47 46.80851064 | |
| 48 47.91666667 | |
| 49 46.93877551 | |
| 50 48.00000000 | |
| 51 47.05882353 | |
| 52 48.07692308 | |
| 53 49.05660377 | |
| 54 48.14814815 | |
| 55 47.27272727 | |
| 56 48.21428571 | |
| 57 49.12280702 | |
| 58 50.00000000 | |
| 59 49.15254237 | |
| 60 50.00000000 | |
| 61 50.81967213 | |
| 62 50.00000000 | |
| 63 50.79365079 | |
| 64 50.00000000 | |
| 65 49.23076923 | |
| 66 50.00000000 | |
| 67 49.25373134 | |
| 68 50.00000000 | |
| 69 50.72463768 | |
| 70 50.00000000 | |
| 71 49.29577465 | |
| 72 48.61111111 | |
| 73 49.31506849 | |
| 74 48.64864865 | |
| 75 49.33333333 | |
| 76 50.00000000 | |
| 77 50.64935065 | |
| 78 51.28205128 | |
| 79 50.63291139 | |
| 80 51.25000000 | |
| 81 50.61728395 | |
| 82 51.21951220 | |
| 83 50.60240964 | |
| 84 51.19047619 | |
| 85 51.76470588 | |
| 86 52.32558140 | |
| 87 52.87356322 | |
| 88 53.40909091 | |
| 89 53.93258427 | |
| 90 54.44444444 | |
| 91 54.94505495 | |
| 92 55.43478261 | |
| 93 54.83870968 | |
| 94 54.25531915 | |
| 95 54.73684211 | |
| 96 54.16666667 | |
| 97 53.60824742 | |
| 98 54.08163265 | |
| 99 54.54545455 | |
| 100 55.00000000 | |
| 101 54.45544554 | |
| 102 54.90196078 | |
| 103 54.36893204 | |
| 104 54.80769231 | |
| 105 55.23809524 | |
| 106 54.71698113 | |
| 107 55.14018692 | |
| 108 54.62962963 | |
| 109 54.12844037 | |
| 110 54.54545455 | |
| 111 54.95495495 | |
| 112 55.35714286 | |
| 113 54.86725664 | |
| 114 54.38596491 | |
| 115 54.78260870 | |
| 116 54.31034483 | |
| 117 54.70085470 | |
| 118 55.08474576 | |
| 119 54.62184874 | |
| 120 55.00000000 | |
| 121 54.54545455 | |
| 122 54.91803279 | |
| 123 55.28455285 | |
| 124 54.83870968 | |
| 125 55.20000000 | |
| 126 55.55555556 | |
| 127 55.90551181 | |
| 128 55.46875000 | |
| 129 55.03875969 | |
| 130 54.61538462 | |
| 131 54.96183206 | |
| 132 55.30303030 | |
| 133 55.63909774 | |
| 134 55.22388060 | |
| 135 54.81481481 | |
| 136 54.41176471 | |
| 137 54.74452555 | |
| 138 55.07246377 | |
| 139 55.39568345 | |
| 140 55.00000000 | |
| 141 55.31914894 | |
| 142 54.92957746 | |
| 143 55.24475524 | |
| 144 54.86111111 | |
| 145 55.17241379 | |
| 146 54.79452055 | |
| 147 55.10204082 | |
| 148 54.72972973 | |
| 149 55.03355705 | |
| 150 55.33333333 | |
| 151 55.62913907 | |
| 152 55.92105263 | |
| 153 56.20915033 | |
| 154 55.84415584 | |
| 155 55.48387097 | |
| 156 55.76923077 | |
| 157 56.05095541 | |
| 158 55.69620253 | |
| 159 55.97484277 | |
| 160 55.62500000 | |
| 161 55.90062112 | |
| 162 56.17283951 | |
| 163 55.82822086 | |
| 164 55.48780488 | |
| 165 55.75757576 | |
| 166 56.02409639 | |
| 167 55.68862275 | |
| 168 55.95238095 | |
| 169 55.62130178 | |
| 170 55.29411765 | |
| 171 55.55555556 | |
| 172 55.23255814 | |
| 173 55.49132948 | |
| 174 55.74712644 | |
| 175 55.42857143 | |
| 176 55.11363636 | |
| 177 55.36723164 | |
| 178 55.05617978 | |
| 179 55.30726257 | |
| 180 55.00000000 | |
| 181 55.24861878 | |
| 182 55.49450549 | |
| 183 55.19125683 | |
| 184 54.89130435 | |
| 185 55.13513514 | |
| 186 55.37634409 | |
| 187 55.08021390 | |
| 188 54.78723404 | |
| 189 54.49735450 | |
| 190 54.21052632 | |
| 191 54.45026178 | |
| 192 54.16666667 | |
| 193 53.88601036 | |
| 194 54.12371134 | |
| 195 54.35897436 | |
| 196 54.59183673 | |
| 197 54.82233503 | |
| 198 55.05050505 | |
| 199 55.27638191 | |
| 200 55.50000000 | |
| 201 55.72139303 | |
| 202 55.94059406 | |
| 203 55.66502463 | |
| 204 55.88235294 | |
| 205 55.60975610 | |
| 206 55.33980583 | |
| 207 55.55555556 | |
| 208 55.76923077 | |
| 209 55.98086124 | |
| 210 56.19047619 | |
| 211 56.39810427 | |
| 212 56.13207547 | |
| 213 55.86854460 | |
| 214 56.07476636 | |
| 215 56.27906977 | |
| 216 56.01851852 | |
| 217 55.76036866 | |
| 218 55.96330275 | |
| 219 56.16438356 | |
| 220 55.90909091 | |
| 221 55.65610860 | |
| 222 55.40540541 | |
| 223 55.15695067 | |
| 224 55.35714286 | |
| 225 55.11111111 | |
| 226 55.30973451 | |
| 227 55.50660793 | |
| 228 55.70175439 | |
| 229 55.89519651 | |
| 230 56.08695652 | |
| 231 55.84415584 | |
| 232 55.60344828 | |
| 233 55.36480687 | |
| 234 55.12820513 | |
| 235 54.89361702 | |
| 236 54.66101695 | |
| 237 54.43037975 | |
| 238 54.62184874 | |
| 239 54.81171548 | |
| 240 55.00000000 | |
| 241 55.18672199 | |
| 242 54.95867769 | |
| 243 54.73251029 | |
| 244 54.50819672 | |
| 245 54.28571429 | |
| 246 54.06504065 | |
| 247 54.25101215 | |
| 248 54.43548387 | |
| 249 54.21686747 | |
| 250 54.40000000 | |
| 251 54.58167331 | |
| 252 54.76190476 | |
| 253 54.54545455 | |
| 254 54.33070866 | |
| 255 54.50980392 | |
| 256 54.68750000 | |
| 257 54.86381323 | |
| 258 54.65116279 | |
| 259 54.44015444 | |
| 260 54.61538462 | |
| 261 54.78927203 | |
| 262 54.96183206 | |
| 263 55.13307985 | |
| 264 55.30303030 | |
| 265 55.09433962 | |
| 266 54.88721805 | |
| 267 55.05617978 | |
| 268 54.85074627 | |
| 269 55.01858736 | |
| 270 55.18518519 | |
| 271 54.98154982 | |
| 272 54.77941176 | |
| 273 54.94505495 | |
| 274 54.74452555 | |
| 275 54.54545455 | |
| 276 54.71014493 | |
| 277 54.87364621 | |
| 278 54.67625899 | |
| 279 54.83870968 | |
| 280 54.64285714 | |
| 281 54.80427046 | |
| 282 54.96453901 | |
| 283 54.77031802 | |
| 284 54.92957746 | |
| 285 54.73684211 | |
| 286 54.89510490 | |
| 287 55.05226481 | |
| 288 55.20833333 | |
| 289 55.01730104 | |
| 290 55.17241379 | |
| 291 55.32646048 | |
| 292 55.13698630 | |
| 293 55.29010239 | |
| 294 55.10204082 | |
| 295 55.25423729 | |
| 296 55.06756757 | |
| 297 55.21885522 | |
| 298 55.03355705 | |
| 299 55.18394649 | |
| 300 55.00000000 | |
| 301 54.81727575 | |
| 302 54.96688742 | |
| 303 54.78547855 | |
| 304 54.93421053 | |
| 305 55.08196721 | |
| 306 54.90196078 | |
| 307 55.04885993 | |
| 308 54.87012987 | |
| 309 55.01618123 | |
| 310 55.16129032 | |
| 311 55.30546624 | |
| 312 55.12820513 | |
| 313 55.27156550 | |
| 314 55.09554140 | |
| 315 54.92063492 | |
| 316 55.06329114 | |
| 317 55.20504732 | |
| 318 55.03144654 | |
| 319 54.85893417 | |
| 320 54.68750000 | |
| 321 54.51713396 | |
| 322 54.65838509 | |
| 323 54.48916409 | |
| 324 54.32098765 | |
| 325 54.15384615 | |
| 326 54.29447853 | |
| 327 54.12844037 | |
| 328 53.96341463 | |
| 329 54.10334347 | |
| 330 54.24242424 | |
| 331 54.38066465 | |
| 332 54.21686747 | |
| 333 54.35435435 | |
| 334 54.19161677 | |
| 335 54.02985075 | |
| 336 53.86904762 | |
| 337 53.70919881 | |
| 338 53.55029586 | |
| 339 53.68731563 | |
| 340 53.82352941 | |
| 341 53.95894428 | |
| 342 53.80116959 | |
| 343 53.93586006 | |
| 344 54.06976744 | |
| 345 53.91304348 | |
| 346 54.04624277 | |
| 347 53.89048991 | |
| 348 53.73563218 | |
| 349 53.86819484 | |
| 350 53.71428571 | |
| 351 53.84615385 | |
| 352 53.69318182 | |
| 353 53.54107649 | |
| 354 53.38983051 | |
| 355 53.23943662 | |
| 356 53.08988764 | |
| 357 52.94117647 | |
| 358 53.07262570 | |
| 359 52.92479109 | |
| 360 53.05555556 | |
| 361 53.18559557 | |
| 362 53.31491713 | |
| 363 53.44352617 | |
| 364 53.29670330 | |
| 365 53.15068493 | |
| 366 53.00546448 | |
| 367 52.86103542 | |
| 368 52.98913043 | |
| 369 52.84552846 | |
| 370 52.70270270 | |
| 371 52.56064690 | |
| 372 52.41935484 | |
| 373 52.27882038 | |
| 374 52.40641711 | |
| 375 52.53333333 | |
| 376 52.65957447 | |
| 377 52.78514589 | |
| 378 52.91005291 | |
| 379 52.77044855 | |
| 380 52.89473684 | |
| 381 52.75590551 | |
| 382 52.61780105 | |
| 383 52.48041775 | |
| 384 52.34375000 | |
| 385 52.46753247 | |
| 386 52.59067358 | |
| 387 52.45478036 | |
| 388 52.31958763 | |
| 389 52.18508997 | |
| 390 52.30769231 | |
| 391 52.17391304 | |
| 392 52.04081633 | |
| 393 52.16284987 | |
| 394 52.28426396 | |
| 395 52.40506329 | |
| 396 52.27272727 | |
| 397 52.39294710 | |
| 398 52.26130653 | |
| 399 52.13032581 | |
| 400 52.25000000 | |
| 401 52.11970075 | |
| 402 52.23880597 | |
| 403 52.35732010 | |
| 404 52.47524752 | |
| 405 52.59259259 | |
| 406 52.70935961 | |
| 407 52.82555283 | |
| 408 52.94117647 | |
| 409 52.81173594 | |
| 410 52.92682927 | |
| 411 53.04136253 | |
| 412 53.15533981 | |
| 413 53.02663438 | |
| 414 52.89855072 | |
| 415 52.77108434 | |
| 416 52.64423077 | |
| 417 52.51798561 | |
| 418 52.39234450 | |
| 419 52.26730310 | |
| 420 52.38095238 | |
| 421 52.25653207 | |
| 422 52.36966825 | |
| 423 52.48226950 | |
| 424 52.35849057 | |
| 425 52.23529412 | |
| 426 52.11267606 | |
| 427 51.99063232 | |
| 428 51.86915888 | |
| 429 51.98135198 | |
| 430 52.09302326 | |
| 431 52.20417633 | |
| 432 52.08333333 | |
| 433 51.96304850 | |
| 434 51.84331797 | |
| 435 51.95402299 | |
| 436 52.06422018 | |
| 437 52.17391304 | |
| 438 52.05479452 | |
| 439 51.93621868 | |
| 440 52.04545455 | |
| 441 52.15419501 | |
| 442 52.03619910 | |
| 443 51.91873589 | |
| 444 51.80180180 | |
| 445 51.91011236 | |
| 446 52.01793722 | |
| 447 52.12527964 | |
| 448 52.23214286 | |
| 449 52.33853007 | |
| 450 52.44444444 | |
| 451 52.32815965 | |
| 452 52.43362832 | |
| 453 52.53863135 | |
| 454 52.42290749 | |
| 455 52.30769231 | |
| 456 52.19298246 | |
| 457 52.07877462 | |
| 458 51.96506550 | |
| 459 52.06971678 | |
| 460 52.17391304 | |
| 461 52.06073753 | |
| 462 52.16450216 | |
| 463 52.05183585 | |
| 464 51.93965517 | |
| 465 52.04301075 | |
| 466 52.14592275 | |
| 467 52.03426124 | |
| 468 51.92307692 | |
| 469 51.81236674 | |
| 470 51.91489362 | |
| 471 52.01698514 | |
| 472 52.11864407 | |
| 473 52.00845666 | |
| 474 51.89873418 | |
| 475 51.78947368 | |
| 476 51.68067227 | |
| 477 51.78197065 | |
| 478 51.88284519 | |
| 479 51.98329854 | |
| 480 51.87500000 | |
| 481 51.76715177 | |
| 482 51.86721992 | |
| 483 51.96687371 | |
| 484 52.06611570 | |
| 485 52.16494845 | |
| 486 52.26337449 | |
| 487 52.36139630 | |
| 488 52.25409836 | |
| 489 52.14723926 | |
| 490 52.04081633 | |
| 491 52.13849287 | |
| 492 52.03252033 | |
| 493 51.92697769 | |
| 494 52.02429150 | |
| 495 51.91919192 | |
| 496 51.81451613 | |
| 497 51.91146881 | |
| 498 52.00803213 | |
| 499 52.10420842 | |
| 500 52.20000000 | |
| 501 52.09580838 | |
| 502 52.19123506 | |
| 503 52.28628231 | |
| 504 52.18253968 | |
| 505 52.27722772 | |
| 506 52.17391304 | |
| 507 52.07100592 | |
| 508 52.16535433 | |
| 509 52.06286837 | |
| 510 52.15686275 | |
| 511 52.05479452 | |
| 512 52.14843750 | |
| 513 52.04678363 | |
| 514 52.14007782 | |
| 515 52.23300971 | |
| 516 52.13178295 | |
| 517 52.03094778 | |
| 518 51.93050193 | |
| 519 51.83044316 | |
| 520 51.73076923 | |
| 521 51.82341651 | |
| 522 51.91570881 | |
| 523 51.81644359 | |
| 524 51.71755725 | |
| 525 51.61904762 | |
| 526 51.52091255 | |
| 527 51.42314991 | |
| 528 51.32575758 | |
| 529 51.22873346 | |
| 530 51.32075472 | |
| 531 51.41242938 | |
| 532 51.50375940 | |
| 533 51.59474672 | |
| 534 51.49812734 | |
| 535 51.40186916 | |
| 536 51.30597015 | |
| 537 51.39664804 | |
| 538 51.30111524 | |
| 539 51.39146568 | |
| 540 51.29629630 | |
| 541 51.20147874 | |
| 542 51.29151292 | |
| 543 51.38121547 | |
| 544 51.28676471 | |
| 545 51.37614679 | |
| 546 51.28205128 | |
| 547 51.18829982 | |
| 548 51.27737226 | |
| 549 51.36612022 | |
| 550 51.45454545 | |
| 551 51.36116152 | |
| 552 51.26811594 | |
| 553 51.35623870 | |
| 554 51.44404332 | |
| 555 51.53153153 | |
| 556 51.43884892 | |
| 557 51.52603232 | |
| 558 51.43369176 | |
| 559 51.52057245 | |
| 560 51.60714286 | |
| 561 51.69340463 | |
| 562 51.77935943 | |
| 563 51.86500888 | |
| 564 51.77304965 | |
| 565 51.68141593 | |
| 566 51.76678445 | |
| 567 51.85185185 | |
| 568 51.93661972 | |
| 569 51.84534271 | |
| 570 51.75438596 | |
| 571 51.66374781 | |
| 572 51.57342657 | |
| 573 51.65794066 | |
| 574 51.74216028 | |
| 575 51.82608696 | |
| 576 51.90972222 | |
| 577 51.99306759 | |
| 578 51.90311419 | |
| 579 51.98618307 | |
| 580 52.06896552 | |
| 581 51.97934596 | |
| 582 51.89003436 | |
| 583 51.97255575 | |
| 584 52.05479452 | |
| 585 52.13675214 | |
| 586 52.21843003 | |
| 587 52.12947189 | |
| 588 52.04081633 | |
| 589 52.12224109 | |
| 590 52.20338983 | |
| 591 52.28426396 | |
| 592 52.36486486 | |
| 593 52.27655987 | |
| 594 52.35690236 | |
| 595 52.43697479 | |
| 596 52.34899329 | |
| 597 52.26130653 | |
| 598 52.17391304 | |
| 599 52.25375626 | |
| 600 52.16666667 | |
| 601 52.07986689 | |
| 602 51.99335548 | |
| 603 51.90713101 | |
| 604 51.82119205 | |
| 605 51.73553719 | |
| 606 51.81518152 | |
| 607 51.72981878 | |
| 608 51.80921053 | |
| 609 51.72413793 | |
| 610 51.80327869 | |
| 611 51.71849427 | |
| 612 51.63398693 | |
| 613 51.54975530 | |
| 614 51.46579805 | |
| 615 51.54471545 | |
| 616 51.62337662 | |
| 617 51.70178282 | |
| 618 51.61812298 | |
| 619 51.53473344 | |
| 620 51.61290323 | |
| 621 51.52979066 | |
| 622 51.44694534 | |
| 623 51.52487961 | |
| 624 51.44230769 | |
| 625 51.36000000 | |
| 626 51.27795527 | |
| 627 51.35566188 | |
| 628 51.27388535 | |
| 629 51.19236884 | |
| 630 51.26984127 | |
| 631 51.18858954 | |
| 632 51.10759494 | |
| 633 51.02685624 | |
| 634 50.94637224 | |
| 635 50.86614173 | |
| 636 50.94339623 | |
| 637 50.86342229 | |
| 638 50.94043887 | |
| 639 50.86071987 | |
| 640 50.78125000 | |
| 641 50.85803432 | |
| 642 50.77881620 | |
| 643 50.69984448 | |
| 644 50.77639752 | |
| 645 50.69767442 | |
| 646 50.61919505 | |
| 647 50.54095827 | |
| 648 50.46296296 | |
| 649 50.38520801 | |
| 650 50.30769231 | |
| 651 50.38402458 | |
| 652 50.30674847 | |
| 653 50.22970904 | |
| 654 50.30581040 | |
| 655 50.22900763 | |
| 656 50.15243902 | |
| 657 50.07610350 | |
| 658 50.15197568 | |
| 659 50.07587253 | |
| 660 50.15151515 | |
| 661 50.07564297 | |
| 662 50.15105740 | |
| 663 50.22624434 | |
| 664 50.30120482 | |
| 665 50.37593985 | |
| 666 50.45045045 | |
| 667 50.52473763 | |
| 668 50.44910180 | |
| 669 50.37369208 | |
| 670 50.44776119 | |
| 671 50.37257824 | |
| 672 50.29761905 | |
| 673 50.22288262 | |
| 674 50.14836795 | |
| 675 50.07407407 | |
| 676 50.00000000 | |
| 677 50.07385524 | |
| 678 50.14749263 | |
| 679 50.22091311 | |
| 680 50.29411765 | |
| 681 50.22026432 | |
| 682 50.14662757 | |
| 683 50.21961933 | |
| 684 50.14619883 | |
| 685 50.21897810 | |
| 686 50.14577259 | |
| 687 50.07278020 | |
| 688 50.14534884 | |
| 689 50.21770682 | |
| 690 50.28985507 | |
| 691 50.36179450 | |
| 692 50.43352601 | |
| 693 50.50505051 | |
| 694 50.57636888 | |
| 695 50.50359712 | |
| 696 50.43103448 | |
| 697 50.50215208 | |
| 698 50.42979943 | |
| 699 50.50071531 | |
| 700 50.57142857 | |
| 701 50.64194009 | |
| 702 50.71225071 | |
| 703 50.64011380 | |
| 704 50.71022727 | |
| 705 50.78014184 | |
| 706 50.84985836 | |
| 707 50.91937765 | |
| 708 50.98870056 | |
| 709 51.05782793 | |
| 710 51.12676056 | |
| 711 51.19549930 | |
| 712 51.26404494 | |
| 713 51.33239832 | |
| 714 51.26050420 | |
| 715 51.18881119 | |
| 716 51.25698324 | |
| 717 51.18549512 | |
| 718 51.11420613 | |
| 719 51.04311544 | |
| 720 51.11111111 | |
| 721 51.04022191 | |
| 722 51.10803324 | |
| 723 51.17565698 | |
| 724 51.24309392 | |
| 725 51.31034483 | |
| 726 51.23966942 | |
| 727 51.16918845 | |
| 728 51.23626374 | |
| 729 51.30315501 | |
| 730 51.36986301 | |
| 731 51.29958960 | |
| 732 51.36612022 | |
| 733 51.43246930 | |
| 734 51.49863760 | |
| 735 51.42857143 | |
| 736 51.49456522 | |
| 737 51.42469471 | |
| 738 51.35501355 | |
| 739 51.42083897 | |
| 740 51.35135135 | |
| 741 51.41700405 | |
| 742 51.48247978 | |
| 743 51.54777927 | |
| 744 51.47849462 | |
| 745 51.54362416 | |
| 746 51.47453083 | |
| 747 51.40562249 | |
| 748 51.33689840 | |
| 749 51.26835781 | |
| 750 51.33333333 | |
| Final result: 51.3333 ±1.8263 | |
| Random chance: 25.0083 ±1.5824 | |