| common_init_from_params: setting dry_penalty_last_n to ctx_size = 768 | |
| common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) | |
| system_info: n_threads = 6 (n_threads_batch = 6) / 12 | Metal : EMBED_LIBRARY = 1 | CPU : NEON = 1 | ARM_FMA = 1 | FP16_VA = 1 | DOTPROD = 1 | LLAMAFILE = 1 | ACCELERATE = 1 | AARCH64_REPACK = 1 | | |
| multiple_choice_score: there are 869 tasks in prompt | |
| multiple_choice_score: selecting 750 random tasks from 869 tasks available | |
| multiple_choice_score: preparing task data...done | |
| multiple_choice_score : calculating TruthfulQA score over 750 tasks. | |
| task acc_norm | |
| 1 100.00000000 | |
| 2 50.00000000 | |
| 3 66.66666667 | |
| 4 50.00000000 | |
| 5 60.00000000 | |
| 6 66.66666667 | |
| 7 57.14285714 | |
| 8 62.50000000 | |
| 9 55.55555556 | |
| 10 50.00000000 | |
| 11 45.45454545 | |
| 12 41.66666667 | |
| 13 38.46153846 | |
| 14 42.85714286 | |
| 15 46.66666667 | |
| 16 43.75000000 | |
| 17 47.05882353 | |
| 18 44.44444444 | |
| 19 42.10526316 | |
| 20 40.00000000 | |
| 21 38.09523810 | |
| 22 40.90909091 | |
| 23 39.13043478 | |
| 24 41.66666667 | |
| 25 40.00000000 | |
| 26 38.46153846 | |
| 27 40.74074074 | |
| 28 39.28571429 | |
| 29 37.93103448 | |
| 30 40.00000000 | |
| 31 38.70967742 | |
| 32 37.50000000 | |
| 33 39.39393939 | |
| 34 41.17647059 | |
| 35 40.00000000 | |
| 36 38.88888889 | |
| 37 40.54054054 | |
| 38 42.10526316 | |
| 39 41.02564103 | |
| 40 42.50000000 | |
| 41 43.90243902 | |
| 42 42.85714286 | |
| 43 44.18604651 | |
| 44 43.18181818 | |
| 45 42.22222222 | |
| 46 43.47826087 | |
| 47 44.68085106 | |
| 48 45.83333333 | |
| 49 44.89795918 | |
| 50 46.00000000 | |
| 51 45.09803922 | |
| 52 46.15384615 | |
| 53 47.16981132 | |
| 54 46.29629630 | |
| 55 45.45454545 | |
| 56 44.64285714 | |
| 57 45.61403509 | |
| 58 46.55172414 | |
| 59 47.45762712 | |
| 60 48.33333333 | |
| 61 49.18032787 | |
| 62 48.38709677 | |
| 63 49.20634921 | |
| 64 48.43750000 | |
| 65 47.69230769 | |
| 66 48.48484848 | |
| 67 47.76119403 | |
| 68 48.52941176 | |
| 69 49.27536232 | |
| 70 48.57142857 | |
| 71 47.88732394 | |
| 72 47.22222222 | |
| 73 47.94520548 | |
| 74 48.64864865 | |
| 75 49.33333333 | |
| 76 50.00000000 | |
| 77 50.64935065 | |
| 78 51.28205128 | |
| 79 50.63291139 | |
| 80 51.25000000 | |
| 81 50.61728395 | |
| 82 51.21951220 | |
| 83 50.60240964 | |
| 84 51.19047619 | |
| 85 51.76470588 | |
| 86 52.32558140 | |
| 87 52.87356322 | |
| 88 53.40909091 | |
| 89 53.93258427 | |
| 90 54.44444444 | |
| 91 54.94505495 | |
| 92 55.43478261 | |
| 93 54.83870968 | |
| 94 54.25531915 | |
| 95 54.73684211 | |
| 96 54.16666667 | |
| 97 53.60824742 | |
| 98 54.08163265 | |
| 99 54.54545455 | |
| 100 55.00000000 | |
| 101 54.45544554 | |
| 102 54.90196078 | |
| 103 54.36893204 | |
| 104 53.84615385 | |
| 105 54.28571429 | |
| 106 53.77358491 | |
| 107 54.20560748 | |
| 108 53.70370370 | |
| 109 53.21100917 | |
| 110 53.63636364 | |
| 111 54.05405405 | |
| 112 54.46428571 | |
| 113 54.86725664 | |
| 114 54.38596491 | |
| 115 54.78260870 | |
| 116 54.31034483 | |
| 117 54.70085470 | |
| 118 55.08474576 | |
| 119 54.62184874 | |
| 120 55.00000000 | |
| 121 54.54545455 | |
| 122 54.91803279 | |
| 123 55.28455285 | |
| 124 54.83870968 | |
| 125 55.20000000 | |
| 126 55.55555556 | |
| 127 55.90551181 | |
| 128 55.46875000 | |
| 129 55.03875969 | |
| 130 54.61538462 | |
| 131 54.96183206 | |
| 132 55.30303030 | |
| 133 55.63909774 | |
| 134 55.22388060 | |
| 135 54.81481481 | |
| 136 54.41176471 | |
| 137 54.74452555 | |
| 138 55.07246377 | |
| 139 55.39568345 | |
| 140 55.71428571 | |
| 141 56.02836879 | |
| 142 56.33802817 | |
| 143 56.64335664 | |
| 144 56.25000000 | |
| 145 56.55172414 | |
| 146 56.16438356 | |
| 147 56.46258503 | |
| 148 56.08108108 | |
| 149 56.37583893 | |
| 150 56.66666667 | |
| 151 56.95364238 | |
| 152 57.23684211 | |
| 153 57.51633987 | |
| 154 57.14285714 | |
| 155 56.77419355 | |
| 156 57.05128205 | |
| 157 57.32484076 | |
| 158 57.59493671 | |
| 159 57.86163522 | |
| 160 57.50000000 | |
| 161 57.76397516 | |
| 162 58.02469136 | |
| 163 57.66871166 | |
| 164 57.31707317 | |
| 165 57.57575758 | |
| 166 57.83132530 | |
| 167 57.48502994 | |
| 168 57.73809524 | |
| 169 57.39644970 | |
| 170 57.64705882 | |
| 171 57.30994152 | |
| 172 56.97674419 | |
| 173 57.22543353 | |
| 174 57.47126437 | |
| 175 57.14285714 | |
| 176 56.81818182 | |
| 177 57.06214689 | |
| 178 56.74157303 | |
| 179 56.98324022 | |
| 180 56.66666667 | |
| 181 56.35359116 | |
| 182 56.59340659 | |
| 183 56.28415301 | |
| 184 55.97826087 | |
| 185 56.21621622 | |
| 186 56.45161290 | |
| 187 56.14973262 | |
| 188 55.85106383 | |
| 189 55.55555556 | |
| 190 55.26315789 | |
| 191 55.49738220 | |
| 192 55.20833333 | |
| 193 54.92227979 | |
| 194 55.15463918 | |
| 195 55.38461538 | |
| 196 55.61224490 | |
| 197 55.83756345 | |
| 198 56.06060606 | |
| 199 56.28140704 | |
| 200 56.50000000 | |
| 201 56.71641791 | |
| 202 56.93069307 | |
| 203 56.65024631 | |
| 204 56.86274510 | |
| 205 56.58536585 | |
| 206 56.31067961 | |
| 207 56.52173913 | |
| 208 56.73076923 | |
| 209 56.93779904 | |
| 210 57.14285714 | |
| 211 57.34597156 | |
| 212 57.07547170 | |
| 213 56.80751174 | |
| 214 57.00934579 | |
| 215 57.20930233 | |
| 216 56.94444444 | |
| 217 56.68202765 | |
| 218 56.88073394 | |
| 219 57.07762557 | |
| 220 56.81818182 | |
| 221 56.56108597 | |
| 222 56.30630631 | |
| 223 56.05381166 | |
| 224 56.25000000 | |
| 225 56.00000000 | |
| 226 55.75221239 | |
| 227 55.94713656 | |
| 228 56.14035088 | |
| 229 56.33187773 | |
| 230 56.52173913 | |
| 231 56.27705628 | |
| 232 56.03448276 | |
| 233 56.22317597 | |
| 234 55.98290598 | |
| 235 55.74468085 | |
| 236 55.50847458 | |
| 237 55.27426160 | |
| 238 55.46218487 | |
| 239 55.64853556 | |
| 240 55.83333333 | |
| 241 56.01659751 | |
| 242 55.78512397 | |
| 243 55.55555556 | |
| 244 55.32786885 | |
| 245 55.10204082 | |
| 246 54.87804878 | |
| 247 55.06072874 | |
| 248 55.24193548 | |
| 249 55.02008032 | |
| 250 55.20000000 | |
| 251 55.37848606 | |
| 252 55.55555556 | |
| 253 55.33596838 | |
| 254 55.11811024 | |
| 255 55.29411765 | |
| 256 55.46875000 | |
| 257 55.64202335 | |
| 258 55.81395349 | |
| 259 55.59845560 | |
| 260 55.76923077 | |
| 261 55.93869732 | |
| 262 56.10687023 | |
| 263 56.27376426 | |
| 264 56.43939394 | |
| 265 56.22641509 | |
| 266 56.01503759 | |
| 267 56.17977528 | |
| 268 55.97014925 | |
| 269 56.13382900 | |
| 270 56.29629630 | |
| 271 56.08856089 | |
| 272 55.88235294 | |
| 273 56.04395604 | |
| 274 55.83941606 | |
| 275 55.63636364 | |
| 276 55.79710145 | |
| 277 55.95667870 | |
| 278 55.75539568 | |
| 279 55.55555556 | |
| 280 55.35714286 | |
| 281 55.51601423 | |
| 282 55.67375887 | |
| 283 55.47703180 | |
| 284 55.63380282 | |
| 285 55.43859649 | |
| 286 55.59440559 | |
| 287 55.74912892 | |
| 288 55.90277778 | |
| 289 55.70934256 | |
| 290 55.86206897 | |
| 291 56.01374570 | |
| 292 55.82191781 | |
| 293 55.97269625 | |
| 294 55.78231293 | |
| 295 55.93220339 | |
| 296 56.08108108 | |
| 297 56.22895623 | |
| 298 56.04026846 | |
| 299 56.18729097 | |
| 300 56.00000000 | |
| 301 55.81395349 | |
| 302 55.96026490 | |
| 303 55.77557756 | |
| 304 55.92105263 | |
| 305 56.06557377 | |
| 306 55.88235294 | |
| 307 56.02605863 | |
| 308 55.84415584 | |
| 309 55.98705502 | |
| 310 56.12903226 | |
| 311 56.27009646 | |
| 312 56.08974359 | |
| 313 56.23003195 | |
| 314 56.05095541 | |
| 315 55.87301587 | |
| 316 56.01265823 | |
| 317 56.15141956 | |
| 318 55.97484277 | |
| 319 55.79937304 | |
| 320 55.62500000 | |
| 321 55.45171340 | |
| 322 55.59006211 | |
| 323 55.41795666 | |
| 324 55.55555556 | |
| 325 55.38461538 | |
| 326 55.52147239 | |
| 327 55.35168196 | |
| 328 55.18292683 | |
| 329 55.31914894 | |
| 330 55.45454545 | |
| 331 55.58912387 | |
| 332 55.42168675 | |
| 333 55.55555556 | |
| 334 55.38922156 | |
| 335 55.22388060 | |
| 336 55.05952381 | |
| 337 54.89614243 | |
| 338 54.73372781 | |
| 339 54.86725664 | |
| 340 55.00000000 | |
| 341 55.13196481 | |
| 342 54.97076023 | |
| 343 55.10204082 | |
| 344 55.23255814 | |
| 345 55.07246377 | |
| 346 55.20231214 | |
| 347 55.04322767 | |
| 348 54.88505747 | |
| 349 55.01432665 | |
| 350 54.85714286 | |
| 351 54.98575499 | |
| 352 54.82954545 | |
| 353 54.67422096 | |
| 354 54.51977401 | |
| 355 54.36619718 | |
| 356 54.21348315 | |
| 357 54.06162465 | |
| 358 54.18994413 | |
| 359 54.03899721 | |
| 360 54.16666667 | |
| 361 54.29362881 | |
| 362 54.41988950 | |
| 363 54.54545455 | |
| 364 54.39560440 | |
| 365 54.24657534 | |
| 366 54.09836066 | |
| 367 53.95095368 | |
| 368 54.07608696 | |
| 369 54.20054201 | |
| 370 54.05405405 | |
| 371 53.90835580 | |
| 372 53.76344086 | |
| 373 53.61930295 | |
| 374 53.74331551 | |
| 375 53.86666667 | |
| 376 53.72340426 | |
| 377 53.84615385 | |
| 378 53.96825397 | |
| 379 53.82585752 | |
| 380 53.94736842 | |
| 381 53.80577428 | |
| 382 53.66492147 | |
| 383 53.52480418 | |
| 384 53.38541667 | |
| 385 53.50649351 | |
| 386 53.62694301 | |
| 387 53.48837209 | |
| 388 53.35051546 | |
| 389 53.21336761 | |
| 390 53.33333333 | |
| 391 53.19693095 | |
| 392 53.06122449 | |
| 393 53.18066158 | |
| 394 53.29949239 | |
| 395 53.41772152 | |
| 396 53.28282828 | |
| 397 53.40050378 | |
| 398 53.26633166 | |
| 399 53.13283208 | |
| 400 53.25000000 | |
| 401 53.11720698 | |
| 402 52.98507463 | |
| 403 53.10173697 | |
| 404 53.21782178 | |
| 405 53.33333333 | |
| 406 53.44827586 | |
| 407 53.56265356 | |
| 408 53.67647059 | |
| 409 53.54523227 | |
| 410 53.65853659 | |
| 411 53.77128954 | |
| 412 53.88349515 | |
| 413 53.75302663 | |
| 414 53.86473430 | |
| 415 53.73493976 | |
| 416 53.60576923 | |
| 417 53.47721823 | |
| 418 53.34928230 | |
| 419 53.22195704 | |
| 420 53.33333333 | |
| 421 53.20665083 | |
| 422 53.31753555 | |
| 423 53.42789598 | |
| 424 53.30188679 | |
| 425 53.17647059 | |
| 426 53.05164319 | |
| 427 52.92740047 | |
| 428 52.80373832 | |
| 429 52.91375291 | |
| 430 53.02325581 | |
| 431 53.13225058 | |
| 432 53.00925926 | |
| 433 52.88683603 | |
| 434 52.76497696 | |
| 435 52.87356322 | |
| 436 52.98165138 | |
| 437 53.08924485 | |
| 438 52.96803653 | |
| 439 53.07517084 | |
| 440 53.18181818 | |
| 441 53.28798186 | |
| 442 53.16742081 | |
| 443 53.04740406 | |
| 444 52.92792793 | |
| 445 53.03370787 | |
| 446 53.13901345 | |
| 447 53.24384787 | |
| 448 53.34821429 | |
| 449 53.45211581 | |
| 450 53.33333333 | |
| 451 53.43680710 | |
| 452 53.53982301 | |
| 453 53.42163355 | |
| 454 53.30396476 | |
| 455 53.18681319 | |
| 456 53.07017544 | |
| 457 52.95404814 | |
| 458 52.83842795 | |
| 459 52.94117647 | |
| 460 53.04347826 | |
| 461 52.92841649 | |
| 462 53.03030303 | |
| 463 52.91576674 | |
| 464 52.80172414 | |
| 465 52.68817204 | |
| 466 52.78969957 | |
| 467 52.67665953 | |
| 468 52.77777778 | |
| 469 52.66524520 | |
| 470 52.76595745 | |
| 471 52.86624204 | |
| 472 52.96610169 | |
| 473 52.85412262 | |
| 474 52.74261603 | |
| 475 52.63157895 | |
| 476 52.52100840 | |
| 477 52.62054507 | |
| 478 52.71966527 | |
| 479 52.81837161 | |
| 480 52.70833333 | |
| 481 52.59875260 | |
| 482 52.69709544 | |
| 483 52.79503106 | |
| 484 52.89256198 | |
| 485 52.98969072 | |
| 486 53.08641975 | |
| 487 53.18275154 | |
| 488 53.27868852 | |
| 489 53.16973415 | |
| 490 53.06122449 | |
| 491 53.15682281 | |
| 492 53.04878049 | |
| 493 52.94117647 | |
| 494 53.03643725 | |
| 495 52.92929293 | |
| 496 52.82258065 | |
| 497 52.91750503 | |
| 498 53.01204819 | |
| 499 53.10621242 | |
| 500 53.20000000 | |
| 501 53.09381238 | |
| 502 53.18725100 | |
| 503 53.28031809 | |
| 504 53.17460317 | |
| 505 53.26732673 | |
| 506 53.16205534 | |
| 507 53.05719921 | |
| 508 53.14960630 | |
| 509 53.04518664 | |
| 510 53.13725490 | |
| 511 53.03326810 | |
| 512 53.12500000 | |
| 513 53.02144250 | |
| 514 53.11284047 | |
| 515 53.20388350 | |
| 516 53.10077519 | |
| 517 52.99806576 | |
| 518 52.89575290 | |
| 519 52.79383430 | |
| 520 52.69230769 | |
| 521 52.78310940 | |
| 522 52.87356322 | |
| 523 52.77246654 | |
| 524 52.67175573 | |
| 525 52.57142857 | |
| 526 52.47148289 | |
| 527 52.37191651 | |
| 528 52.27272727 | |
| 529 52.17391304 | |
| 530 52.26415094 | |
| 531 52.35404896 | |
| 532 52.44360902 | |
| 533 52.53283302 | |
| 534 52.43445693 | |
| 535 52.33644860 | |
| 536 52.23880597 | |
| 537 52.32774674 | |
| 538 52.23048327 | |
| 539 52.31910946 | |
| 540 52.22222222 | |
| 541 52.12569316 | |
| 542 52.21402214 | |
| 543 52.30202578 | |
| 544 52.20588235 | |
| 545 52.29357798 | |
| 546 52.19780220 | |
| 547 52.10237660 | |
| 548 52.18978102 | |
| 549 52.27686703 | |
| 550 52.18181818 | |
| 551 52.08711434 | |
| 552 51.99275362 | |
| 553 52.07956600 | |
| 554 52.16606498 | |
| 555 52.25225225 | |
| 556 52.15827338 | |
| 557 52.24416517 | |
| 558 52.15053763 | |
| 559 52.23613596 | |
| 560 52.32142857 | |
| 561 52.40641711 | |
| 562 52.49110320 | |
| 563 52.57548845 | |
| 564 52.48226950 | |
| 565 52.38938053 | |
| 566 52.47349823 | |
| 567 52.55731922 | |
| 568 52.64084507 | |
| 569 52.54833040 | |
| 570 52.45614035 | |
| 571 52.36427320 | |
| 572 52.27272727 | |
| 573 52.35602094 | |
| 574 52.43902439 | |
| 575 52.52173913 | |
| 576 52.60416667 | |
| 577 52.68630849 | |
| 578 52.76816609 | |
| 579 52.84974093 | |
| 580 52.93103448 | |
| 581 52.83993115 | |
| 582 52.74914089 | |
| 583 52.83018868 | |
| 584 52.91095890 | |
| 585 52.99145299 | |
| 586 53.07167235 | |
| 587 52.98126065 | |
| 588 52.89115646 | |
| 589 52.97113752 | |
| 590 53.05084746 | |
| 591 53.13028765 | |
| 592 53.20945946 | |
| 593 53.11973019 | |
| 594 53.19865320 | |
| 595 53.27731092 | |
| 596 53.18791946 | |
| 597 53.09882747 | |
| 598 53.01003344 | |
| 599 53.08848080 | |
| 600 53.00000000 | |
| 601 52.91181364 | |
| 602 52.82392027 | |
| 603 52.73631841 | |
| 604 52.64900662 | |
| 605 52.56198347 | |
| 606 52.64026403 | |
| 607 52.55354201 | |
| 608 52.63157895 | |
| 609 52.54515599 | |
| 610 52.62295082 | |
| 611 52.53682488 | |
| 612 52.45098039 | |
| 613 52.36541599 | |
| 614 52.28013029 | |
| 615 52.35772358 | |
| 616 52.43506494 | |
| 617 52.51215559 | |
| 618 52.42718447 | |
| 619 52.34248788 | |
| 620 52.41935484 | |
| 621 52.33494364 | |
| 622 52.25080386 | |
| 623 52.32744783 | |
| 624 52.24358974 | |
| 625 52.16000000 | |
| 626 52.07667732 | |
| 627 52.15311005 | |
| 628 52.22929936 | |
| 629 52.30524642 | |
| 630 52.38095238 | |
| 631 52.45641838 | |
| 632 52.37341772 | |
| 633 52.29067930 | |
| 634 52.20820189 | |
| 635 52.12598425 | |
| 636 52.20125786 | |
| 637 52.11930926 | |
| 638 52.19435737 | |
| 639 52.11267606 | |
| 640 52.03125000 | |
| 641 52.10608424 | |
| 642 52.02492212 | |
| 643 51.94401244 | |
| 644 52.01863354 | |
| 645 51.93798450 | |
| 646 51.85758514 | |
| 647 51.77743431 | |
| 648 51.69753086 | |
| 649 51.61787365 | |
| 650 51.53846154 | |
| 651 51.61290323 | |
| 652 51.53374233 | |
| 653 51.45482389 | |
| 654 51.52905199 | |
| 655 51.45038168 | |
| 656 51.37195122 | |
| 657 51.29375951 | |
| 658 51.36778116 | |
| 659 51.28983308 | |
| 660 51.36363636 | |
| 661 51.28593041 | |
| 662 51.35951662 | |
| 663 51.43288084 | |
| 664 51.50602410 | |
| 665 51.57894737 | |
| 666 51.65165165 | |
| 667 51.72413793 | |
| 668 51.64670659 | |
| 669 51.56950673 | |
| 670 51.64179104 | |
| 671 51.56482861 | |
| 672 51.48809524 | |
| 673 51.41158990 | |
| 674 51.33531157 | |
| 675 51.25925926 | |
| 676 51.18343195 | |
| 677 51.25553914 | |
| 678 51.32743363 | |
| 679 51.39911635 | |
| 680 51.47058824 | |
| 681 51.39500734 | |
| 682 51.31964809 | |
| 683 51.24450952 | |
| 684 51.16959064 | |
| 685 51.24087591 | |
| 686 51.16618076 | |
| 687 51.09170306 | |
| 688 51.16279070 | |
| 689 51.08853411 | |
| 690 51.15942029 | |
| 691 51.23010130 | |
| 692 51.30057803 | |
| 693 51.37085137 | |
| 694 51.44092219 | |
| 695 51.51079137 | |
| 696 51.43678161 | |
| 697 51.36298422 | |
| 698 51.28939828 | |
| 699 51.21602289 | |
| 700 51.28571429 | |
| 701 51.21255350 | |
| 702 51.28205128 | |
| 703 51.20910384 | |
| 704 51.27840909 | |
| 705 51.34751773 | |
| 706 51.41643059 | |
| 707 51.48514851 | |
| 708 51.55367232 | |
| 709 51.62200282 | |
| 710 51.69014085 | |
| 711 51.75808720 | |
| 712 51.82584270 | |
| 713 51.89340813 | |
| 714 51.82072829 | |
| 715 51.74825175 | |
| 716 51.81564246 | |
| 717 51.74337517 | |
| 718 51.67130919 | |
| 719 51.59944367 | |
| 720 51.66666667 | |
| 721 51.59500693 | |
| 722 51.52354571 | |
| 723 51.59059474 | |
| 724 51.65745856 | |
| 725 51.72413793 | |
| 726 51.65289256 | |
| 727 51.58184319 | |
| 728 51.64835165 | |
| 729 51.57750343 | |
| 730 51.64383562 | |
| 731 51.70998632 | |
| 732 51.77595628 | |
| 733 51.84174625 | |
| 734 51.90735695 | |
| 735 51.83673469 | |
| 736 51.90217391 | |
| 737 51.83175034 | |
| 738 51.76151762 | |
| 739 51.82679296 | |
| 740 51.75675676 | |
| 741 51.82186235 | |
| 742 51.88679245 | |
| 743 51.95154778 | |
| 744 51.88172043 | |
| 745 51.94630872 | |
| 746 51.87667560 | |
| 747 51.80722892 | |
| 748 51.73796791 | |
| 749 51.66889186 | |
| 750 51.73333333 | |
| Final result: 51.7333 ±1.8259 | |
| Random chance: 25.0083 ±1.5824 | |