update sc-cot scores

src/detail_math_score.json

@@ -1,5 +1,5 @@
 {
-    "time": "2025-01-23 11:23:17",
+    "time": "2025-01-24 15:10:27",
     "results": {
         "IO": {
             "gpt-3.5-turbo": {
@@ -971,7 +971,7 @@
                 },
                 "gsm8k": {
                     "Score": 9.62,
-                    "Pass rate": 0.169,
+                    "Pass rate": 0.1691,
                     "Cost($)": 0.0,
                     "Framework": "",
                     "X-shot": "8.0",
@@ -1337,30 +1337,30 @@
                     "Eval Date": "2025/1/7"
                 },
                 "gsm8k": {
-                    "Score": 82.56,
-                    "Pass rate": 0.9985,
-                    "Cost($)": 2.6285,
+                    "Score": 79.91,
+                    "Pass rate": 0.9992,
+                    "Cost($)": 3.3938,
                     "Framework": "",
                     "X-shot": "8.0",
                     "Samples": 1319,
-                    "All tokens": 2560697,
-                    "Total input tokens": 1212520,
-                    "Average input tokens": 919,
-                    "Total output tokens": 1348177,
-                    "Average output tokens": 1022
+                    "All tokens": 4089612,
+                    "Total input tokens": 2740652,
+                    "Average input tokens": 2078,
+                    "Total output tokens": 1348960,
+                    "Average output tokens": 1023
                 },
                 "AQuA": {
-                    "Score": 70.47,
-                    "Pass rate": 0.9882,
-                    "Cost($)": 0.5578,
+                    "Score": 66.14,
+                    "Pass rate": 0.9921,
+                    "Cost($)": 0.7888,
                     "Framework": "",
                     "X-shot": "0.0",
                     "Samples": 254,
-                    "All tokens": 418617,
-                    "Total input tokens": 70157,
-                    "Average input tokens": 276,
-                    "Total output tokens": 348460,
-                    "Average output tokens": 1372
+                    "All tokens": 847335,
+                    "Total input tokens": 482192,
+                    "Average input tokens": 1898,
+                    "Total output tokens": 365143,
+                    "Average output tokens": 1438
                 }
             },
             "Doubao-lite-32k": {
@@ -1370,30 +1370,30 @@
                     "Eval Date": "2025/1/7"
                 },
                 "gsm8k": {
-                    "Score": 83.7,
-                    "Pass rate": 0.997,
-                    "Cost($)": 0.155,
+                    "Score": 87.26,
+                    "Pass rate": 0.9992,
+                    "Cost($)": 0.2083,
                     "Framework": "",
                     "X-shot": "8.0",
                     "Samples": 1319,
-                    "All tokens": 2507687,
-                    "Total input tokens": 1230019,
-                    "Average input tokens": 933,
-                    "Total output tokens": 1277668,
-                    "Average output tokens": 969
+                    "All tokens": 3888813,
+                    "Total input tokens": 2691714,
+                    "Average input tokens": 2041,
+                    "Total output tokens": 1197099,
+                    "Average output tokens": 908
                 },
                 "AQuA": {
-                    "Score": 81.5,
-                    "Pass rate": 0.9764,
-                    "Cost($)": 0.0347,
+                    "Score": 81.1,
+                    "Pass rate": 0.9724,
+                    "Cost($)": 0.0519,
                     "Framework": "",
                     "X-shot": "0.0",
                     "Samples": 254,
-                    "All tokens": 465846,
-                    "Total input tokens": 83830,
-                    "Average input tokens": 330,
-                    "Total output tokens": 382016,
-                    "Average output tokens": 1504
+                    "All tokens": 885986,
+                    "Total input tokens": 503751,
+                    "Average input tokens": 1983,
+                    "Total output tokens": 382235,
+                    "Average output tokens": 1505
                 }
             },
             "gpt-4o": {
@@ -1403,30 +1403,30 @@
                     "Eval Date": "2025/1/22"
                 },
                 "gsm8k": {
-                    "Score": 90.75,
-                    "Pass rate": 1.0,
-                    "Cost($)": 24.2428,
+                    "Score": 90.3,
+                    "Pass rate": 0.9992,
+                    "Cost($)": 31.0542,
                     "Framework": "",
                     "X-shot": "8.0",
                     "Samples": 1319,
-                    "All tokens": 3300971,
-                    "Total input tokens": 1168927,
-                    "Average input tokens": 886,
-                    "Total output tokens": 2132044,
-                    "Average output tokens": 1616
+                    "All tokens": 5798173,
+                    "Total input tokens": 3590336,
+                    "Average input tokens": 2722,
+                    "Total output tokens": 2207837,
+                    "Average output tokens": 1674
                 },
                 "AQuA": {
-                    "Score": 88.19,
-                    "Pass rate": 1.0,
-                    "Cost($)": 6.2412,
+                    "Score": 86.61,
+                    "Pass rate": 0.9882,
+                    "Cost($)": 8.1485,
                     "Framework": "",
                     "X-shot": "0.0",
                     "Samples": 254,
-                    "All tokens": 678811,
-                    "Total input tokens": 72916,
-                    "Average input tokens": 287,
-                    "Total output tokens": 605895,
-                    "Average output tokens": 2385
+                    "All tokens": 1373206,
+                    "Total input tokens": 744478,
+                    "Average input tokens": 2931,
+                    "Total output tokens": 628728,
+                    "Average output tokens": 2475
                 }
             },
             "Qwen2.5-72B-Instruct": {
@@ -1436,30 +1436,30 @@
                     "Eval Date": "2025/1/22"
                 },
                 "gsm8k": {
-                    "Score": 90.67,
+                    "Score": 93.86,
                     "Pass rate": 1.0,
-                    "Cost($)": 4.2651,
+                    "Cost($)": 5.9858,
                     "Framework": "",
                     "X-shot": "8.0",
                     "Samples": 1319,
-                    "All tokens": 7565637,
-                    "Total input tokens": 5292383,
-                    "Average input tokens": 4012,
-                    "Total output tokens": 2273254,
-                    "Average output tokens": 1723
+                    "All tokens": 10618008,
+                    "Total input tokens": 8136223,
+                    "Average input tokens": 6168,
+                    "Total output tokens": 2481785,
+                    "Average output tokens": 1882
                 },
                 "AQuA": {
-                    "Score": 85.83,
-                    "Pass rate": 0.9843,
-                    "Cost($)": 0.5576,
+                    "Score": 85.04,
+                    "Pass rate": 0.9921,
+                    "Cost($)": 1.0348,
                     "Framework": "",
                     "X-shot": "0.0",
                     "Samples": 254,
-                    "All tokens": 989058,
-                    "Total input tokens": 241149,
-                    "Average input tokens": 949,
-                    "Total output tokens": 747909,
-                    "Average output tokens": 2945
+                    "All tokens": 1835669,
+                    "Total input tokens": 1051218,
+                    "Average input tokens": 4139,
+                    "Total output tokens": 784451,
+                    "Average output tokens": 3088
                 }
             },
             "Llama-3.3-70B-Instruct": {
@@ -1469,30 +1469,30 @@
                     "Eval Date": "2025/1/22"
                 },
                 "gsm8k": {
-                    "Score": 95.45,
+                    "Score": 95.07,
                     "Pass rate": 1.0,
-                    "Cost($)": 4.5021,
+                    "Cost($)": 6.2005,
                     "Framework": "",
                     "X-shot": "8.0",
                     "Samples": 1319,
-                    "All tokens": 7985996,
-                    "Total input tokens": 5406763,
-                    "Average input tokens": 4099,
-                    "Total output tokens": 2579233,
-                    "Average output tokens": 1955
+                    "All tokens": 10998794,
+                    "Total input tokens": 8413717,
+                    "Average input tokens": 6379,
+                    "Total output tokens": 2585077,
+                    "Average output tokens": 1960
                 },
                 "AQuA": {
-                    "Score": 86.61,
+                    "Score": 82.28,
                     "Pass rate": 0.9921,
-                    "Cost($)": 0.5847,
+                    "Cost($)": 1.0756,
                     "Framework": "",
                     "X-shot": "0.0",
                     "Samples": 254,
-                    "All tokens": 1037124,
-                    "Total input tokens": 283248,
-                    "Average input tokens": 1115,
-                    "Total output tokens": 753876,
-                    "Average output tokens": 2968
+                    "All tokens": 1907924,
+                    "Total input tokens": 1135251,
+                    "Average input tokens": 4469,
+                    "Total output tokens": 772673,
+                    "Average output tokens": 3042
                 }
             },
             "Qwen2.5-7B-Instruct": {
@@ -1502,30 +1502,30 @@
                     "Eval Date": "2025/1/22"
                 },
                 "gsm8k": {
-                    "Score": 88.32,
-                    "Pass rate": 0.9985,
+                    "Score": 91.13,
+                    "Pass rate": 1.0,
                     "Cost($)": 0.0,
                     "Framework": "",
                     "X-shot": "8.0",
                     "Samples": 1319,
-                    "All tokens": 8173818,
-                    "Total input tokens": 5668252,
-                    "Average input tokens": 4297,
-                    "Total output tokens": 2505566,
-                    "Average output tokens": 1900
+                    "All tokens": 11140985,
+                    "Total input tokens": 8586888,
+                    "Average input tokens": 6510,
+                    "Total output tokens": 2554097,
+                    "Average output tokens": 1936
                 },
                 "AQuA": {
-                    "Score": 81.5,
+                    "Score": 79.92,
                     "Pass rate": 1.0,
                     "Cost($)": 0.0,
                     "Framework": "",
                     "X-shot": "0.0",
                     "Samples": 254,
-                    "All tokens": 1015368,
-                    "Total input tokens": 278848,
-                    "Average input tokens": 1098,
-                    "Total output tokens": 736520,
-                    "Average output tokens": 2900
+                    "All tokens": 1845332,
+                    "Total input tokens": 1098280,
+                    "Average input tokens": 4324,
+                    "Total output tokens": 747052,
+                    "Average output tokens": 2941
                 }
             },
             "Llama-3.1-8B-Instruct": {
@@ -1535,30 +1535,30 @@
                     "Eval Date": "2025/1/22"
                 },
                 "gsm8k": {
-                    "Score": 75.21,
+                    "Score": 73.46,
                     "Pass rate": 0.9955,
                     "Cost($)": 0.0,
                     "Framework": "",
                     "X-shot": "8.0",
                     "Samples": 1319,
-                    "All tokens": 8444203,
-                    "Total input tokens": 5334657,
-                    "Average input tokens": 4044,
-                    "Total output tokens": 3109546,
-                    "Average output tokens": 2358
+                    "All tokens": 11778716,
+                    "Total input tokens": 8630514,
+                    "Average input tokens": 6543,
+                    "Total output tokens": 3148202,
+                    "Average output tokens": 2387
                 },
                 "AQuA": {
-                    "Score": 53.15,
-                    "Pass rate": 0.9606,
+                    "Score": 59.45,
+                    "Pass rate": 0.9724,
                     "Cost($)": 0.0,
                     "Framework": "",
                     "X-shot": "0.0",
                     "Samples": 254,
-                    "All tokens": 1041346,
-                    "Total input tokens": 372968,
-                    "Average input tokens": 1468,
-                    "Total output tokens": 668378,
-                    "Average output tokens": 2631
+                    "All tokens": 1651333,
+                    "Total input tokens": 971003,
+                    "Average input tokens": 3823,
+                    "Total output tokens": 680330,
+                    "Average output tokens": 2678
                 }
             },
             "Internllm2_5-7B": {
@@ -1568,30 +1568,30 @@
                     "Eval Date": "2025/1/22"
                 },
                 "gsm8k": {
-                    "Score": 41.39,
-                    "Pass rate": 0.9826,
+                    "Score": 48.22,
+                    "Pass rate": 0.9841,
                     "Cost($)": 0.0,
                     "Framework": "",
                     "X-shot": "8.0",
                     "Samples": 1319,
-                    "All tokens": 10024857,
-                    "Total input tokens": 6674518,
-                    "Average input tokens": 5060,
-                    "Total output tokens": 3350339,
-                    "Average output tokens": 2540
+                    "All tokens": 14526431,
+                    "Total input tokens": 10678792,
+                    "Average input tokens": 8096,
+                    "Total output tokens": 3847639,
+                    "Average output tokens": 2917
                 },
                 "AQuA": {
-                    "Score": 35.85,
-                    "Pass rate": 0.988,
+                    "Score": 39.37,
+                    "Pass rate": 0.9803,
                     "Cost($)": 0.0,
                     "Framework": "",
                     "X-shot": "0.0",
                     "Samples": 254,
-                    "All tokens": 1240388,
-                    "Total input tokens": 530701,
-                    "Average input tokens": 2089,
-                    "Total output tokens": 709687,
-                    "Average output tokens": 2794
+                    "All tokens": 2296222,
+                    "Total input tokens": 1420494,
+                    "Average input tokens": 5592,
+                    "Total output tokens": 875728,
+                    "Average output tokens": 3448
                 }
             },
             "Qwen2-1.5B-Instruct": {
@@ -1601,30 +1601,30 @@
                     "Eval Date": "2025/1/22"
                 },
                 "gsm8k": {
-                    "Score": 5.53,
-                    "Pass rate": 0.8673,
+                    "Score": 11.75,
+                    "Pass rate": 0.9189,
                     "Cost($)": 0.0,
                     "Framework": "",
                     "X-shot": "8.0",
                     "Samples": 1319,
-                    "All tokens": 8961768,
-                    "Total input tokens": 5844218,
-                    "Average input tokens": 4431,
-                    "Total output tokens": 3117550,
-                    "Average output tokens": 2364
+                    "All tokens": 12411942,
+                    "Total input tokens": 9066115,
+                    "Average input tokens": 6873,
+                    "Total output tokens": 3345827,
+                    "Average output tokens": 2537
                 },
                 "AQuA": {
-                    "Score": 30.31,
-                    "Pass rate": 0.9724,
+                    "Score": 23.62,
+                    "Pass rate": 0.9646,
                     "Cost($)": 0.0,
                     "Framework": "",
                     "X-shot": "0.0",
                     "Samples": 254,
-                    "All tokens": 1157076,
-                    "Total input tokens": 430703,
-                    "Average input tokens": 1696,
-                    "Total output tokens": 726373,
-                    "Average output tokens": 2860
+                    "All tokens": 1775335,
+                    "Total input tokens": 1034362,
+                    "Average input tokens": 4072,
+                    "Total output tokens": 740973,
+                    "Average output tokens": 2917
                 }
             },
             "Qwen2-0.5B-Instruct": {
@@ -1634,30 +1634,30 @@
                     "Eval Date": "2025/1/22"
                 },
                 "gsm8k": {
-                    "Score": 3.79,
-                    "Pass rate": 0.9484,
+                    "Score": 1.67,
+                    "Pass rate": 0.9469,
                     "Cost($)": 0.0,
                     "Framework": "",
                     "X-shot": "8.0",
                     "Samples": 1319,
-                    "All tokens": 10533815,
-                    "Total input tokens": 6529832,
-                    "Average input tokens": 4951,
-                    "Total output tokens": 4003983,
-                    "Average output tokens": 3036
+                    "All tokens": 16465720,
+                    "Total input tokens": 11019864,
+                    "Average input tokens": 8355,
+                    "Total output tokens": 5445856,
+                    "Average output tokens": 4129
                 },
                 "AQuA": {
-                    "Score": 30.71,
-                    "Pass rate": 0.9843,
+                    "Score": 22.83,
+                    "Pass rate": 0.9724,
                     "Cost($)": 0.0,
                     "Framework": "",
                     "X-shot": "0.0",
                     "Samples": 254,
-                    "All tokens": 1225539,
-                    "Total input tokens": 496206,
-                    "Average input tokens": 1954,
-                    "Total output tokens": 729333,
-                    "Average output tokens": 2871
+                    "All tokens": 2215091,
+                    "Total input tokens": 1246929,
+                    "Average input tokens": 4909,
+                    "Total output tokens": 968162,
+                    "Average output tokens": 3812
                 }
             }
         }

src/detail_results.csv

@@ -1,16 +1,16 @@
 Rank,Algorithm,Dataset,LLM,Eval Date,Score,Pass rate,X-shot,Cost($),Framework,Samples,All tokens,Total input tokens,Average input tokens,Total output tokens,Average output tokens
-1,SC-CoT,AQuA,gpt-4o,2025/1/22,88.19,1.0,0.0,6.2412,,254,678811,72916,287,605895,2385
-2,SC-CoT,AQuA,Llama-3.3-70B-Instruct,2025/1/22,86.61,0.9921,0.0,0.5847,,254,1037124,283248,1115,753876,2968
-3,CoT,AQuA,Qwen2.5-72B-Instruct,2025/1/22,86.22,0.9921,0.0,0.0808,,254,143289,25143,99,118146,465
-4,SC-CoT,AQuA,Qwen2.5-72B-Instruct,2025/1/22,85.83,0.9843,0.0,0.5576,,254,989058,241149,949,747909,2945
-5,IO,AQuA,Qwen2.5-72B-Instruct,2025/1/22,84.25,0.9961,0.0,0.0742,,254,131604,25397,100,106207,418
-6,CoT,AQuA,Llama-3.3-70B-Instruct,2025/1/22,83.46,0.9843,0.0,0.0927,,254,164389,32555,128,131834,519
-7,IO,AQuA,Llama-3.3-70B-Instruct,2025/1/22,82.68,0.9921,0.0,0.0798,,254,141567,32809,129,108758,428
-8,CoT,AQuA,Doubao-lite-32k,2025/1/7,82.68,0.9724,0.0,0.0066,,254,94577,27978,110,66599,262
-9,CoT,AQuA,gpt-4o,2025/1/22,82.68,0.9803,0.0,1.0417,,254,123017,25123,99,97894,385
-10,SC-CoT,AQuA,Doubao-lite-32k,2025/1/7,81.5,0.9764,0.0,0.0347,,254,465846,83830,330,382016,1504
-11,SC-CoT,AQuA,Qwen2.5-7B-Instruct,2025/1/22,81.5,1.0,0.0,0.0,,254,1015368,278848,1098,736520,2900
-12,CoT,AQuA,Qwen2.5-7B-Instruct,2025/1/22,80.71,0.9961,0.0,0.0,,254,149736,33017,130,116719,460
+1,SC-CoT,AQuA,gpt-4o,2025/1/22,86.61,0.9882,0.0,8.1485,,254,1373206,744478,2931,628728,2475
+2,CoT,AQuA,Qwen2.5-72B-Instruct,2025/1/22,86.22,0.9921,0.0,0.0808,,254,143289,25143,99,118146,465
+3,SC-CoT,AQuA,Qwen2.5-72B-Instruct,2025/1/22,85.04,0.9921,0.0,1.0348,,254,1835669,1051218,4139,784451,3088
+4,IO,AQuA,Qwen2.5-72B-Instruct,2025/1/22,84.25,0.9961,0.0,0.0742,,254,131604,25397,100,106207,418
+5,CoT,AQuA,Llama-3.3-70B-Instruct,2025/1/22,83.46,0.9843,0.0,0.0927,,254,164389,32555,128,131834,519
+6,IO,AQuA,Llama-3.3-70B-Instruct,2025/1/22,82.68,0.9921,0.0,0.0798,,254,141567,32809,129,108758,428
+7,CoT,AQuA,Doubao-lite-32k,2025/1/7,82.68,0.9724,0.0,0.0066,,254,94577,27978,110,66599,262
+8,CoT,AQuA,gpt-4o,2025/1/22,82.68,0.9803,0.0,1.0417,,254,123017,25123,99,97894,385
+9,SC-CoT,AQuA,Llama-3.3-70B-Instruct,2025/1/22,82.28,0.9921,0.0,1.0756,,254,1907924,1135251,4469,772673,3042
+10,SC-CoT,AQuA,Doubao-lite-32k,2025/1/7,81.1,0.9724,0.0,0.0519,,254,885986,503751,1983,382235,1505
+11,CoT,AQuA,Qwen2.5-7B-Instruct,2025/1/22,80.71,0.9961,0.0,0.0,,254,149736,33017,130,116719,460
+12,SC-CoT,AQuA,Qwen2.5-7B-Instruct,2025/1/22,79.92,1.0,0.0,0.0,,254,1845332,1098280,4324,747052,2941
 13,PoT,AQuA,Llama-3.3-70B-Instruct,2025/1/22,79.53,0.9921,0.0,0.1746,,254,309799,240735,948,69064,272
 14,IO,AQuA,Doubao-lite-32k,2025/1/7,79.13,1.0,0.0,0.0058,,254,87742,33058,130,54684,215
 15,ReAct-Pro*,AQuA,Llama-3.3-70B-Instruct,2025/1/22,79.13,0.9961,0.0,0.768,,254,1362379,1119143,4406,243236,958
@@ -22,60 +22,60 @@ Rank,Algorithm,Dataset,LLM,Eval Date,Score,Pass rate,X-shot,Cost($),Framework,Sa
 21,ReAct-Pro*,AQuA,Qwen2.5-7B-Instruct,2025/1/22,74.41,0.9921,0.0,0.0,,254,695844,564165,2221,131679,518
 22,ReAct-Pro*,AQuA,Qwen2.5-72B-Instruct,2025/1/22,73.23,1.0,0.0,0.3177,,254,563603,441765,1739,121838,480
 23,PoT,AQuA,Doubao-lite-32k,2025/1/7,71.65,0.9685,0.0,0.0147,,254,309436,259863,1023,49573,195
-24,SC-CoT,AQuA,gpt-3.5-turbo,2025/1/7,70.47,0.9882,0.0,0.5578,,254,418617,70157,276,348460,1372
-25,PoT,AQuA,Qwen2.5-7B-Instruct,2025/1/22,68.11,1.0,0.0,0.0,,254,313728,264517,1041,49211,194
+24,PoT,AQuA,Qwen2.5-7B-Instruct,2025/1/22,68.11,1.0,0.0,0.0,,254,313728,264517,1041,49211,194
+25,SC-CoT,AQuA,gpt-3.5-turbo,2025/1/7,66.14,0.9921,0.0,0.7888,,254,847335,482192,1898,365143,1438
 26,ReAct-Pro*,AQuA,gpt-3.5-turbo,2025/1/7,64.57,0.9803,0.0,0.4928,,254,903587,862614,3396,40973,161
 27,CoT,AQuA,gpt-3.5-turbo,2025/1/7,61.02,0.937,0.0,0.0957,,254,80793,25447,100,55346,218
 28,CoT,AQuA,Llama-3.1-8B-Instruct,2025/1/22,60.63,1.0,0.0,0.0,,254,144435,32555,128,111880,440
 29,PoT,AQuA,gpt-3.5-turbo,2025/1/7,59.45,1.0,0.0,0.1748,,254,266654,225162,886,41492,163
-30,ReAct-Pro*,AQuA,gpt-4o,2025/1/22,57.48,0.9724,0.0,2.304,,254,692096,615589,2424,76507,301
-31,ReAct-Pro*,AQuA,Llama-3.1-8B-Instruct,2025/1/22,55.51,0.9685,0.0,0.0,,254,4340821,3764723,14822,576098,2268
-32,SC-CoT,AQuA,Llama-3.1-8B-Instruct,2025/1/22,53.15,0.9606,0.0,0.0,,254,1041346,372968,1468,668378,2631
+30,SC-CoT,AQuA,Llama-3.1-8B-Instruct,2025/1/22,59.45,0.9724,0.0,0.0,,254,1651333,971003,3823,680330,2678
+31,ReAct-Pro*,AQuA,gpt-4o,2025/1/22,57.48,0.9724,0.0,2.304,,254,692096,615589,2424,76507,301
+32,ReAct-Pro*,AQuA,Llama-3.1-8B-Instruct,2025/1/22,55.51,0.9685,0.0,0.0,,254,4340821,3764723,14822,576098,2268
 33,CoT,AQuA,Internllm2_5-7B,2025/1/22,52.76,0.8937,0.0,0.0,,254,127520,26610,105,100910,397
 34,IO,AQuA,Llama-3.1-8B-Instruct,2025/1/22,51.18,0.9882,0.0,0.0,,254,133106,26459,104,106647,420
 35,IO,AQuA,Internllm2_5-7B,2025/1/22,47.64,0.9094,0.0,0.0,,254,185041,50232,198,134809,531
 36,ReAct-Pro*,AQuA,Internllm2_5-7B,2025/1/22,40.94,0.9685,0.0,0.0,,254,4428801,3592039,14142,836762,3294
 37,CoT,AQuA,Qwen2-1.5B-Instruct,2025/1/22,40.55,0.9882,0.0,0.0,,254,110040,30477,120,79563,313
-38,IO,AQuA,gpt-3.5-turbo,2025/1/7,38.98,1.0,0.0,0.038,,254,42471,25701,101,16770,66
-39,PoT,AQuA,Llama-3.1-8B-Instruct,2025/1/22,36.61,0.9685,0.0,0.0,,254,290914,240613,947,50301,198
-40,PoT,AQuA,Internllm2_5-7B,2025/1/22,36.61,0.9882,0.0,0.0,,254,301962,233505,919,68457,270
-41,SC-CoT,AQuA,Internllm2_5-7B,2025/1/22,35.85,0.988,0.0,0.0,,254,1240388,530701,2089,709687,2794
+38,SC-CoT,AQuA,Internllm2_5-7B,2025/1/22,39.37,0.9803,0.0,0.0,,254,2296222,1420494,5592,875728,3448
+39,IO,AQuA,gpt-3.5-turbo,2025/1/7,38.98,1.0,0.0,0.038,,254,42471,25701,101,16770,66
+40,PoT,AQuA,Llama-3.1-8B-Instruct,2025/1/22,36.61,0.9685,0.0,0.0,,254,290914,240613,947,50301,198
+41,PoT,AQuA,Internllm2_5-7B,2025/1/22,36.61,0.9882,0.0,0.0,,254,301962,233505,919,68457,270
 42,CoT,AQuA,Qwen2-0.5B-Instruct,2025/1/22,33.07,0.9882,0.0,0.0,,254,117339,30477,120,86862,342
 43,PoT,AQuA,Qwen2-1.5B-Instruct,2025/1/22,30.71,0.9646,0.0,0.0,,254,298475,246560,971,51915,204
-44,SC-CoT,AQuA,Qwen2-0.5B-Instruct,2025/1/22,30.71,0.9843,0.0,0.0,,254,1225539,496206,1954,729333,2871
-45,SC-CoT,AQuA,Qwen2-1.5B-Instruct,2025/1/22,30.31,0.9724,0.0,0.0,,254,1157076,430703,1696,726373,2860
-46,IO,AQuA,Qwen2-1.5B-Instruct,2025/1/22,29.13,0.9764,0.0,0.0,,254,71047,27937,110,43110,170
-47,IO,AQuA,Qwen2-0.5B-Instruct,2025/1/22,27.17,0.9882,0.0,0.0,,254,110415,27937,110,82478,325
-48,ReAct-Pro*,AQuA,Qwen2-1.5B-Instruct,2025/1/22,25.59,0.9606,0.0,0.0,,254,5072004,4555858,17936,516146,2032
-49,ReAct-Pro*,AQuA,Qwen2-0.5B-Instruct,2025/1/22,24.02,0.9685,0.0,0.0,,254,7170087,6344167,24977,825920,3252
+44,IO,AQuA,Qwen2-1.5B-Instruct,2025/1/22,29.13,0.9764,0.0,0.0,,254,71047,27937,110,43110,170
+45,IO,AQuA,Qwen2-0.5B-Instruct,2025/1/22,27.17,0.9882,0.0,0.0,,254,110415,27937,110,82478,325
+46,ReAct-Pro*,AQuA,Qwen2-1.5B-Instruct,2025/1/22,25.59,0.9606,0.0,0.0,,254,5072004,4555858,17936,516146,2032
+47,ReAct-Pro*,AQuA,Qwen2-0.5B-Instruct,2025/1/22,24.02,0.9685,0.0,0.0,,254,7170087,6344167,24977,825920,3252
+48,SC-CoT,AQuA,Qwen2-1.5B-Instruct,2025/1/22,23.62,0.9646,0.0,0.0,,254,1775335,1034362,4072,740973,2917
+49,SC-CoT,AQuA,Qwen2-0.5B-Instruct,2025/1/22,22.83,0.9724,0.0,0.0,,254,2215091,1246929,4909,968162,3812
 50,PoT,AQuA,Qwen2-0.5B-Instruct,2025/1/22,17.32,0.9213,0.0,0.0,,254,322281,258867,1019,63414,250
-1,SC-CoT,gsm8k,Llama-3.3-70B-Instruct,2025/1/22,95.45,1.0,8.0,4.5021,,1319,7985996,5406763,4099,2579233,1955
+1,SC-CoT,gsm8k,Llama-3.3-70B-Instruct,2025/1/22,95.07,1.0,8.0,6.2005,,1319,10998794,8413717,6379,2585077,1960
 2,CoT,gsm8k,gpt-4o,2025/1/22,94.09,1.0,8.0,4.5367,,1319,1165166,948668,719,216498,164
 3,CoT,gsm8k,Llama-3.3-70B-Instruct,2025/1/22,93.93,1.0,8.0,0.687,,1319,1218665,990168,751,228497,173
-4,PoT,gsm8k,gpt-4o,2025/1/22,93.1,0.9977,8.0,4.2166,,1319,1247912,1101672,835,146240,111
-5,CoT,gsm8k,Qwen2.5-72B-Instruct,2025/1/22,92.87,1.0,8.0,0.7195,,1319,1276252,1005119,762,271133,206
-6,PoT,gsm8k,Qwen2.5-72B-Instruct,2025/1/22,92.34,0.9939,8.0,0.7054,,1319,1251210,1106682,839,144528,110
-7,IO,gsm8k,Llama-3.3-70B-Instruct,2025/1/22,92.27,1.0,8.0,0.4709,,1319,835275,583916,443,251359,191
-8,SC-CoT,gsm8k,gpt-4o,2025/1/22,90.75,1.0,8.0,24.2428,,1319,3300971,1168927,886,2132044,1616
-9,SC-CoT,gsm8k,Qwen2.5-72B-Instruct,2025/1/22,90.67,1.0,8.0,4.2651,,1319,7565637,5292383,4012,2273254,1723
-10,CoT,gsm8k,Doubao-lite-32k,2025/1/7,89.31,1.0,8.0,0.0558,,1319,1201820,1042095,790,159725,121
-11,IO,gsm8k,gpt-4o,2025/1/22,88.4,1.0,8.0,3.3463,,1319,741446,542416,411,199030,151
-12,SC-CoT,gsm8k,Qwen2.5-7B-Instruct,2025/1/22,88.32,0.9985,8.0,0.0,,1319,8173818,5668252,4297,2505566,1900
+4,SC-CoT,gsm8k,Qwen2.5-72B-Instruct,2025/1/22,93.86,1.0,8.0,5.9858,,1319,10618008,8136223,6168,2481785,1882
+5,PoT,gsm8k,gpt-4o,2025/1/22,93.1,0.9977,8.0,4.2166,,1319,1247912,1101672,835,146240,111
+6,CoT,gsm8k,Qwen2.5-72B-Instruct,2025/1/22,92.87,1.0,8.0,0.7195,,1319,1276252,1005119,762,271133,206
+7,PoT,gsm8k,Qwen2.5-72B-Instruct,2025/1/22,92.34,0.9939,8.0,0.7054,,1319,1251210,1106682,839,144528,110
+8,IO,gsm8k,Llama-3.3-70B-Instruct,2025/1/22,92.27,1.0,8.0,0.4709,,1319,835275,583916,443,251359,191
+9,SC-CoT,gsm8k,Qwen2.5-7B-Instruct,2025/1/22,91.13,1.0,8.0,0.0,,1319,11140985,8586888,6510,2554097,1936
+10,SC-CoT,gsm8k,gpt-4o,2025/1/22,90.3,0.9992,8.0,31.0542,,1319,5798173,3590336,2722,2207837,1674
+11,CoT,gsm8k,Doubao-lite-32k,2025/1/7,89.31,1.0,8.0,0.0558,,1319,1201820,1042095,790,159725,121
+12,IO,gsm8k,gpt-4o,2025/1/22,88.4,1.0,8.0,3.3463,,1319,741446,542416,411,199030,151
 13,ReAct-Pro*,gsm8k,Llama-3.3-70B-Instruct,2025/1/22,87.64,0.9992,8.0,10.1124,,1319,17937864,17038928,12918,898936,682
 14,ReAct-Pro*,gsm8k,Qwen2.5-72B-Instruct,2025/1/22,87.26,1.0,8.0,10.5479,,1319,18710437,18160983,13769,549454,417
-15,IO,gsm8k,Qwen2.5-72B-Instruct,2025/1/22,86.58,1.0,8.0,0.4899,,1319,869060,555340,421,313720,238
-16,CoT,gsm8k,Qwen2.5-7B-Instruct,2025/1/22,85.67,1.0,8.0,0.0,,1319,1290805,1046008,793,244797,186
-17,ReAct-Pro*,gsm8k,Doubao-lite-32k,2025/1/7,85.6,0.9962,8.0,0.2512,,1319,5998639,5862016,4444,136623,104
-18,SC-CoT,gsm8k,Doubao-lite-32k,2025/1/7,83.7,0.997,8.0,0.155,,1319,2507687,1230019,933,1277668,969
+15,SC-CoT,gsm8k,Doubao-lite-32k,2025/1/7,87.26,0.9992,8.0,0.2083,,1319,3888813,2691714,2041,1197099,908
+16,IO,gsm8k,Qwen2.5-72B-Instruct,2025/1/22,86.58,1.0,8.0,0.4899,,1319,869060,555340,421,313720,238
+17,CoT,gsm8k,Qwen2.5-7B-Instruct,2025/1/22,85.67,1.0,8.0,0.0,,1319,1290805,1046008,793,244797,186
+18,ReAct-Pro*,gsm8k,Doubao-lite-32k,2025/1/7,85.6,0.9962,8.0,0.2512,,1319,5998639,5862016,4444,136623,104
 19,ReAct-Pro*,gsm8k,Qwen2.5-7B-Instruct,2025/1/22,82.87,1.0,8.0,0.0,,1319,14850914,14355752,10884,495162,375
-20,SC-CoT,gsm8k,gpt-3.5-turbo,2025/1/7,82.56,0.9985,8.0,2.6285,,1319,2560697,1212520,919,1348177,1022
+20,SC-CoT,gsm8k,gpt-3.5-turbo,2025/1/7,79.91,0.9992,8.0,3.3938,,1319,4089612,2740652,2078,1348960,1023
 21,PoT,gsm8k,Doubao-lite-32k,2025/1/7,79.61,0.9257,8.0,0.0576,,1319,1288055,1170038,887,118017,89
 22,CoT,gsm8k,gpt-3.5-turbo,2025/1/7,78.7,1.0,8.0,0.6788,,1319,1088041,953242,723,134799,102
 23,CoT,gsm8k,Internllm2_5-7B,2025/1/22,77.71,0.997,8.0,0.0,,1319,1202163,968163,734,234000,177
 24,PoT,gsm8k,gpt-3.5-turbo,2025/1/7,76.88,0.9924,8.0,0.6902,,1319,1187080,1090418,827,96662,73
 25,CoT,gsm8k,Llama-3.1-8B-Instruct,2025/1/22,75.44,0.9992,8.0,0.0,,1319,1248329,990168,751,258161,196
-26,SC-CoT,gsm8k,Llama-3.1-8B-Instruct,2025/1/22,75.21,0.9955,8.0,0.0,,1319,8444203,5334657,4044,3109546,2358
-27,ReAct-Pro*,gsm8k,gpt-3.5-turbo,2025/1/7,74.91,0.9939,8.0,3.4633,,1319,6646286,6506164,4933,140122,106
+26,ReAct-Pro*,gsm8k,gpt-3.5-turbo,2025/1/7,74.91,0.9939,8.0,3.4633,,1319,6646286,6506164,4933,140122,106
+27,SC-CoT,gsm8k,Llama-3.1-8B-Instruct,2025/1/22,73.46,0.9955,8.0,0.0,,1319,11778716,8630514,6543,3148202,2387
 28,PoT,gsm8k,Llama-3.3-70B-Instruct,2025/1/22,73.09,0.7961,8.0,0.9736,,1319,1727044,1126025,854,601019,456
 29,IO,gsm8k,Doubao-lite-32k,2025/1/7,72.02,0.9992,8.0,0.0354,,1319,740483,617377,468,123106,93
 30,ReAct-Pro*,gsm8k,Llama-3.1-8B-Instruct,2025/1/22,67.78,0.9856,8.0,0.0,,1319,22835767,21044978,15955,1790789,1358
@@ -84,7 +84,7 @@ Rank,Algorithm,Dataset,LLM,Eval Date,Score,Pass rate,X-shot,Cost($),Framework,Sa
 33,IO,gsm8k,Qwen2.5-7B-Instruct,2025/1/22,57.24,1.0,8.0,0.0,,1319,887913,596229,452,291684,221
 34,IO,gsm8k,Llama-3.1-8B-Instruct,2025/1/22,57.16,0.9955,8.0,0.0,,1319,1745429,550941,418,1194488,906
 35,CoT,gsm8k,Qwen2-1.5B-Instruct,2025/1/22,55.5,1.0,8.0,0.0,,1319,1218525,1032818,783,185707,141
-36,SC-CoT,gsm8k,Internllm2_5-7B,2025/1/22,41.39,0.9826,8.0,0.0,,1319,10024857,6674518,5060,3350339,2540
+36,SC-CoT,gsm8k,Internllm2_5-7B,2025/1/22,48.22,0.9841,8.0,0.0,,1319,14526431,10678792,8096,3847639,2917
 37,PoT,gsm8k,Llama-3.1-8B-Instruct,2025/1/22,38.67,0.5542,8.0,0.0,,1319,1391111,1147538,870,243573,185
 38,PoT,gsm8k,Internllm2_5-7B,2025/1/22,38.21,0.489,8.0,0.0,,1319,1324949,1136843,862,188106,143
 39,IO,gsm8k,gpt-3.5-turbo,2025/1/7,37.83,0.9992,8.0,0.3328,,1319,586553,546990,415,39563,30
@@ -94,8 +94,8 @@ Rank,Algorithm,Dataset,LLM,Eval Date,Score,Pass rate,X-shot,Cost($),Framework,Sa
 43,PoT,gsm8k,Qwen2-1.5B-Instruct,2025/1/22,18.5,0.3101,8.0,0.0,,1319,1327522,1151528,873,175994,133
 44,IO,gsm8k,Qwen2-1.5B-Instruct,2025/1/22,16.68,1.0,8.0,0.0,,1319,736996,568530,431,168466,128
 45,IO,gsm8k,Qwen2-0.5B-Instruct,2025/1/22,14.71,1.0,8.0,0.0,,1319,834897,568116,431,266781,202
-46,IO,gsm8k,Internllm2_5-7B,2025/1/22,11.6,0.9795,8.0,0.0,,1319,1113728,679302,515,434426,329
-47,PoT,gsm8k,Qwen2-0.5B-Instruct,2025/1/22,9.62,0.169,8.0,0.0,,1319,1389135,1151528,873,237607,180
-48,ReAct-Pro*,gsm8k,Qwen2-0.5B-Instruct,2025/1/22,7.66,0.9522,8.0,0.0,,1319,55392611,52431343,39751,2961268,2245
-49,SC-CoT,gsm8k,Qwen2-1.5B-Instruct,2025/1/22,5.53,0.8673,8.0,0.0,,1319,8961768,5844218,4431,3117550,2364
-50,SC-CoT,gsm8k,Qwen2-0.5B-Instruct,2025/1/22,3.79,0.9484,8.0,0.0,,1319,10533815,6529832,4951,4003983,3036
+46,SC-CoT,gsm8k,Qwen2-1.5B-Instruct,2025/1/22,11.75,0.9189,8.0,0.0,,1319,12411942,9066115,6873,3345827,2537
+47,IO,gsm8k,Internllm2_5-7B,2025/1/22,11.6,0.9795,8.0,0.0,,1319,1113728,679302,515,434426,329
+48,PoT,gsm8k,Qwen2-0.5B-Instruct,2025/1/22,9.62,0.1691,8.0,0.0,,1319,1389135,1151528,873,237607,180
+49,ReAct-Pro*,gsm8k,Qwen2-0.5B-Instruct,2025/1/22,7.66,0.9522,8.0,0.0,,1319,55392611,52431343,39751,2961268,2245
+50,SC-CoT,gsm8k,Qwen2-0.5B-Instruct,2025/1/22,1.67,0.9469,8.0,0.0,,1319,16465720,11019864,8355,5445856,4129

src/overall_math_score.json

@@ -1,5 +1,5 @@
 {
-    "time": "2025-01-23 11:23:17",
+    "time": "2025-01-24 15:10:27",
     "results": {
         "IO": {
             "META": {
@@ -68,12 +68,12 @@
                 "Eval Date": "2025/1/7"
             },
             "gsm8k": {
-                "Score": 82.56,
-                "Cost($)": 2.6285
+                "Score": 79.91,
+                "Cost($)": 3.3938
             },
             "AQuA": {
-                "Score": 70.47,
-                "Cost($)": 0.5578
+                "Score": 66.14,
+                "Cost($)": 0.7888
             }
         },
         "IO-Doubao-lite-32k": {
@@ -143,12 +143,12 @@
                 "Eval Date": "2025/1/7"
             },
             "gsm8k": {
-                "Score": 83.7,
-                "Cost($)": 0.155
+                "Score": 87.26,
+                "Cost($)": 0.2083
             },
             "AQuA": {
-                "Score": 81.5,
-                "Cost($)": 0.0347
+                "Score": 81.1,
+                "Cost($)": 0.0519
             }
         },
         "IO-gpt-4o": {
@@ -218,12 +218,12 @@
                 "Eval Date": "2025/1/22"
             },
             "gsm8k": {
-                "Score": 90.75,
-                "Cost($)": 24.2428
+                "Score": 90.3,
+                "Cost($)": 31.0542
             },
             "AQuA": {
-                "Score": 88.19,
-                "Cost($)": 6.2412
+                "Score": 86.61,
+                "Cost($)": 8.1485
             }
         },
         "IO-Qwen2.5-72B-Instruct": {
@@ -293,12 +293,12 @@
                 "Eval Date": "2025/1/22"
             },
             "gsm8k": {
-                "Score": 90.67,
-                "Cost($)": 4.2651
+                "Score": 93.86,
+                "Cost($)": 5.9858
             },
             "AQuA": {
-                "Score": 85.83,
-                "Cost($)": 0.5576
+                "Score": 85.04,
+                "Cost($)": 1.0348
             }
         },
         "IO-Llama-3.3-70B-Instruct": {
@@ -368,12 +368,12 @@
                 "Eval Date": "2025/1/22"
             },
             "gsm8k": {
-                "Score": 95.45,
-                "Cost($)": 4.5021
+                "Score": 95.07,
+                "Cost($)": 6.2005
             },
             "AQuA": {
-                "Score": 86.61,
-                "Cost($)": 0.5847
+                "Score": 82.28,
+                "Cost($)": 1.0756
             }
         },
         "IO-Qwen2.5-7B-Instruct": {
@@ -443,11 +443,11 @@
                 "Eval Date": "2025/1/22"
             },
             "gsm8k": {
-                "Score": 88.32,
+                "Score": 91.13,
                 "Cost($)": 0.0
             },
             "AQuA": {
-                "Score": 81.5,
+                "Score": 79.92,
                 "Cost($)": 0.0
             }
         },
@@ -518,11 +518,11 @@
                 "Eval Date": "2025/1/22"
             },
             "gsm8k": {
-                "Score": 75.21,
+                "Score": 73.46,
                 "Cost($)": 0.0
             },
             "AQuA": {
-                "Score": 53.15,
+                "Score": 59.45,
                 "Cost($)": 0.0
             }
         },
@@ -593,11 +593,11 @@
                 "Eval Date": "2025/1/22"
             },
             "gsm8k": {
-                "Score": 41.39,
+                "Score": 48.22,
                 "Cost($)": 0.0
             },
             "AQuA": {
-                "Score": 35.85,
+                "Score": 39.37,
                 "Cost($)": 0.0
             }
         },
@@ -668,11 +668,11 @@
                 "Eval Date": "2025/1/22"
             },
             "gsm8k": {
-                "Score": 5.53,
+                "Score": 11.75,
                 "Cost($)": 0.0
             },
             "AQuA": {
-                "Score": 30.31,
+                "Score": 23.62,
                 "Cost($)": 0.0
             }
         },
@@ -743,11 +743,11 @@
                 "Eval Date": "2025/1/22"
             },
             "gsm8k": {
-                "Score": 3.79,
+                "Score": 1.67,
                 "Cost($)": 0.0
             },
             "AQuA": {
-                "Score": 30.71,
+                "Score": 22.83,
                 "Cost($)": 0.0
             }
         }

src/overall_results.csv

@@ -1,40 +1,40 @@
 Rank,Algorithm,LLM,Eval Date,Avg Score,gsm8k-Score,gsm8k-Cost($),AQuA-Score,AQuA-Cost($)
-1.0,SC-CoT,Llama-3.3-70B-Instruct,2025/1/22,91.03,95.45,4.5021,86.61,0.5847
-2.0,CoT,Qwen2.5-72B-Instruct,2025/1/22,89.55,92.87,0.7195,86.22,0.0808
-3.0,SC-CoT,gpt-4o,2025/1/22,89.47,90.75,24.2428,88.19,6.2412
-4.0,CoT,Llama-3.3-70B-Instruct,2025/1/22,88.70,93.93,0.687,83.46,0.0927
-5.0,CoT,gpt-4o,2025/1/22,88.39,94.09,4.5367,82.68,1.0417
-6.0,SC-CoT,Qwen2.5-72B-Instruct,2025/1/22,88.25,90.67,4.2651,85.83,0.5576
+1.0,CoT,Qwen2.5-72B-Instruct,2025/1/22,89.55,92.87,0.7195,86.22,0.0808
+2.0,SC-CoT,Qwen2.5-72B-Instruct,2025/1/22,89.45,93.86,5.9858,85.04,1.0348
+3.0,CoT,Llama-3.3-70B-Instruct,2025/1/22,88.70,93.93,0.687,83.46,0.0927
+4.0,SC-CoT,Llama-3.3-70B-Instruct,2025/1/22,88.68,95.07,6.2005,82.28,1.0756
+5.0,SC-CoT,gpt-4o,2025/1/22,88.46,90.3,31.0542,86.61,8.1485
+6.0,CoT,gpt-4o,2025/1/22,88.39,94.09,4.5367,82.68,1.0417
 7.0,IO,Llama-3.3-70B-Instruct,2025/1/22,87.48,92.27,0.4709,82.68,0.0798
 8.0,CoT,Doubao-lite-32k,2025/1/7,86.00,89.31,0.0558,82.68,0.0066
-9.0,IO,Qwen2.5-72B-Instruct,2025/1/22,85.42,86.58,0.4899,84.25,0.0742
-10.0,SC-CoT,Qwen2.5-7B-Instruct,2025/1/22,84.91,88.32,0.0,81.5,0.0
-11.0,PoT,gpt-4o,2025/1/22,84.15,93.1,4.2166,75.2,1.6087
-12.0,PoT,Qwen2.5-72B-Instruct,2025/1/22,83.77,92.34,0.7054,75.2,0.1645
-13.0,ReAct-Pro*,Llama-3.3-70B-Instruct,2025/1/22,83.39,87.64,10.1124,79.13,0.768
-14.0,CoT,Qwen2.5-7B-Instruct,2025/1/22,83.19,85.67,0.0,80.71,0.0
-15.0,SC-CoT,Doubao-lite-32k,2025/1/7,82.60,83.7,0.155,81.5,0.0347
+9.0,SC-CoT,Qwen2.5-7B-Instruct,2025/1/22,85.53,91.13,0.0,79.92,0.0
+10.0,IO,Qwen2.5-72B-Instruct,2025/1/22,85.42,86.58,0.4899,84.25,0.0742
+11.0,SC-CoT,Doubao-lite-32k,2025/1/7,84.18,87.26,0.2083,81.1,0.0519
+12.0,PoT,gpt-4o,2025/1/22,84.15,93.1,4.2166,75.2,1.6087
+13.0,PoT,Qwen2.5-72B-Instruct,2025/1/22,83.77,92.34,0.7054,75.2,0.1645
+14.0,ReAct-Pro*,Llama-3.3-70B-Instruct,2025/1/22,83.39,87.64,10.1124,79.13,0.768
+15.0,CoT,Qwen2.5-7B-Instruct,2025/1/22,83.19,85.67,0.0,80.71,0.0
 16.0,IO,gpt-4o,2025/1/22,82.00,88.4,3.3463,75.59,1.1453
 17.0,ReAct-Pro*,Doubao-lite-32k,2025/1/7,81.58,85.6,0.2512,77.56,0.0445
 18.0,ReAct-Pro*,Qwen2.5-72B-Instruct,2025/1/22,80.25,87.26,10.5479,73.23,0.3177
 19.0,ReAct-Pro*,Qwen2.5-7B-Instruct,2025/1/22,78.64,82.87,0.0,74.41,0.0
-20.0,SC-CoT,gpt-3.5-turbo,2025/1/7,76.52,82.56,2.6285,70.47,0.5578
-21.0,PoT,Llama-3.3-70B-Instruct,2025/1/22,76.31,73.09,0.9736,79.53,0.1746
-22.0,PoT,Doubao-lite-32k,2025/1/7,75.63,79.61,0.0576,71.65,0.0147
-23.0,IO,Doubao-lite-32k,2025/1/7,75.58,72.02,0.0354,79.13,0.0058
+20.0,PoT,Llama-3.3-70B-Instruct,2025/1/22,76.31,73.09,0.9736,79.53,0.1746
+21.0,PoT,Doubao-lite-32k,2025/1/7,75.63,79.61,0.0576,71.65,0.0147
+22.0,IO,Doubao-lite-32k,2025/1/7,75.58,72.02,0.0354,79.13,0.0058
+23.0,SC-CoT,gpt-3.5-turbo,2025/1/7,73.03,79.91,3.3938,66.14,0.7888
 24.0,CoT,gpt-3.5-turbo,2025/1/7,69.86,78.7,0.6788,61.02,0.0957
 25.0,ReAct-Pro*,gpt-3.5-turbo,2025/1/7,69.74,74.91,3.4633,64.57,0.4928
 26.0,PoT,gpt-3.5-turbo,2025/1/7,68.17,76.88,0.6902,59.45,0.1748
 27.0,CoT,Llama-3.1-8B-Instruct,2025/1/22,68.04,75.44,0.0,60.63,0.0
 28.0,IO,Qwen2.5-7B-Instruct,2025/1/22,67.99,57.24,0.0,78.74,0.0
-29.0,CoT,Internllm2_5-7B,2025/1/22,65.24,77.71,0.0,52.76,0.0
-30.0,SC-CoT,Llama-3.1-8B-Instruct,2025/1/22,64.18,75.21,0.0,53.15,0.0
+29.0,SC-CoT,Llama-3.1-8B-Instruct,2025/1/22,66.46,73.46,0.0,59.45,0.0
+30.0,CoT,Internllm2_5-7B,2025/1/22,65.24,77.71,0.0,52.76,0.0
 31.0,PoT,Qwen2.5-7B-Instruct,2025/1/22,63.47,58.83,0.0,68.11,0.0
 32.0,ReAct-Pro*,Llama-3.1-8B-Instruct,2025/1/22,61.65,67.78,0.0,55.51,0.0
 33.0,ReAct-Pro*,gpt-4o,2025/1/22,60.40,63.31,39.0751,57.48,2.304
 34.0,IO,Llama-3.1-8B-Instruct,2025/1/22,54.17,57.16,0.0,51.18,0.0
 35.0,CoT,Qwen2-1.5B-Instruct,2025/1/22,48.03,55.5,0.0,40.55,0.0
-36.0,SC-CoT,Internllm2_5-7B,2025/1/22,38.62,41.39,0.0,35.85,0.0
+36.0,SC-CoT,Internllm2_5-7B,2025/1/22,43.80,48.22,0.0,39.37,0.0
 37.0,IO,gpt-3.5-turbo,2025/1/7,38.41,37.83,0.3328,38.98,0.038
 38.0,PoT,Llama-3.1-8B-Instruct,2025/1/22,37.64,38.67,0.0,36.61,0.0
 39.0,PoT,Internllm2_5-7B,2025/1/22,37.41,38.21,0.0,36.61,0.0
@@ -45,7 +45,7 @@ Rank,Algorithm,LLM,Eval Date,Avg Score,gsm8k-Score,gsm8k-Cost($),AQuA-Score,AQuA
 44.0,PoT,Qwen2-1.5B-Instruct,2025/1/22,24.61,18.5,0.0,30.71,0.0
 45.0,IO,Qwen2-1.5B-Instruct,2025/1/22,22.91,16.68,0.0,29.13,0.0
 46.0,IO,Qwen2-0.5B-Instruct,2025/1/22,20.94,14.71,0.0,27.17,0.0
-47.0,SC-CoT,Qwen2-1.5B-Instruct,2025/1/22,17.92,5.53,0.0,30.31,0.0
-48.0,SC-CoT,Qwen2-0.5B-Instruct,2025/1/22,17.25,3.79,0.0,30.71,0.0
-49.0,ReAct-Pro*,Qwen2-0.5B-Instruct,2025/1/22,15.84,7.66,0.0,24.02,0.0
-50.0,PoT,Qwen2-0.5B-Instruct,2025/1/22,13.47,9.62,0.0,17.32,0.0
+47.0,SC-CoT,Qwen2-1.5B-Instruct,2025/1/22,17.69,11.75,0.0,23.62,0.0
+48.0,ReAct-Pro*,Qwen2-0.5B-Instruct,2025/1/22,15.84,7.66,0.0,24.02,0.0
+49.0,PoT,Qwen2-0.5B-Instruct,2025/1/22,13.47,9.62,0.0,17.32,0.0
+50.0,SC-CoT,Qwen2-0.5B-Instruct,2025/1/22,12.25,1.67,0.0,22.83,0.0

src/record.csv

@@ -28,7 +28,7 @@ PoT,gsm8k,2025/1/22,Qwen2.5-7B-Instruct,58.83,70.51,8,,1319,"1,145,390",868,"217
 PoT,gsm8k,2025/1/22,Llama-3.1-8B-Instruct,38.67,55.42,8,,1319,"1,147,538",870,"243,573",185,"1,391,111",0.0000,,,,,,,,,,,,,,,,,,,,
 PoT,gsm8k,2025/1/22,Internllm2_5-7B,38.21,48.9,8,,1319,"1,136,843",862,"188,106",143,"1,324,949",0.0000,,,,,,,,,,,,,,,,,,,,
 PoT,gsm8k,2025/1/22,Qwen2-1.5B-Instruct,18.5,31.01,8,,1319,"1,151,528",873,"175,994",133,"1,327,522",0.0000,,,,,,,,,,,,,,,,,,,,
-PoT,gsm8k,2025/1/22,Qwen2-0.5B-Instruct,9.62,16.9,8,,1319,"1,151,528",873,"237,607",180,"1,389,135",0.0000,,,,,,,,,,,,,,,,,,,,
+PoT,gsm8k,2025/1/22,Qwen2-0.5B-Instruct,9.62,16.91,8,,1319,"1,151,528",873,"237,607",180,"1,389,135",0.0000,,,,,,,,,,,,,,,,,,,,
 CoT,gsm8k,2025/1/7,gpt-3.5-turbo,78.7,100,8,,1319,"953,242",723,"134,799",102,"1,088,041",0.6788,,,,,,,,,,,,,,,,,,,,
 CoT,gsm8k,2025/1/7,Doubao-lite-32k,89.31,100,8,,1319,"1,042,095",790,"159,725",121,"1,201,820",0.0558,0.4084635 （元）,,,,,,,,,,,,,,,,,,,
 CoT,gsm8k,2025/1/22,gpt-4o,94.09,100,8,,1319,"948,668",719,"216,498",164,"1,165,166",4.5367,,,,,,,,,,,,,,,,,,,,
@@ -39,16 +39,16 @@ CoT,gsm8k,2025/1/22,Llama-3.1-8B-Instruct,75.44,99.92,8,,1319,"990,168",751,"258
 CoT,gsm8k,2025/1/22,Internllm2_5-7B,77.71,99.7,8,,1319,"968,163",734,"234,000",177,"1,202,163",0.0000,,,,,,,,,,,,,,,,,,,,
 CoT,gsm8k,2025/1/22,Qwen2-1.5B-Instruct,55.5,100,8,,1319,"1,032,818",783,"185,707",141,"1,218,525",0.0000,,,,,,,,,,,,,,,,,,,,
 CoT,gsm8k,2025/1/22,Qwen2-0.5B-Instruct,35.94,99.92,8,,1319,"1,032,818",783,"190,641",145,"1,223,459",0.0000,,,,,,,,,,,,,,,,,,,,
-SC-CoT,gsm8k,2025/1/7,gpt-3.5-turbo,82.56,99.85,8,"temperature=1, path_num=5",1319,"1,212,520",919,"1,348,177","1,022","2,560,697",2.6285,,,,,,,,,,,,,,,,,,,,
-SC-CoT,gsm8k,2025/1/7,Doubao-lite-32k,83.7,99.7,8,"temperature=1, path_num=5",1319,"1,230,019",933,"1,277,668",969,"2,507,687",0.1550,,,,,,,,,,,,,,,,,,,,
-SC-CoT,gsm8k,2025/1/22,gpt-4o,90.75,100,8,"temperature=1, path_num=5",1319,"1,168,927",886,"2,132,044","1,616","3,300,971",24.2428,,,,,,,,,,,,,,,,,,,,
-SC-CoT,gsm8k,2025/1/22,Qwen2.5-72B-Instruct,90.67,100,8,"temperature=1, path_num=5",1319,"5,292,383","4,012","2,273,254","1,723","7,565,637",4.2651,,,,,,,,,,,,,,,,,,,,
-SC-CoT,gsm8k,2025/1/22,Llama-3.3-70B-Instruct,95.45,100,8,"temperature=1, path_num=5",1319,"5,406,763","4,099","2,579,233","1,955","7,985,996",4.5021,,,,,,,,,,,,,,,,,,,,
-SC-CoT,gsm8k,2025/1/22,Qwen2.5-7B-Instruct,88.32,99.85,8,"temperature=1, path_num=5",1319,"5,668,252","4,297","2,505,566","1,900","8,173,818",0.0000,,,,,,,,,,,,,,,,,,,,
-SC-CoT,gsm8k,2025/1/22,Llama-3.1-8B-Instruct,75.21,99.55,8,"temperature=1, path_num=5",1319,"5,334,657","4,044","3,109,546","2,358","8,444,203",0.0000,,,,,,,,,,,,,,,,,,,,
-SC-CoT,gsm8k,2025/1/22,Internllm2_5-7B,41.39,98.26,8,"temperature=1, path_num=5",1319,"6,674,518","5,060","3,350,339","2,540","10,024,857",0.0000,,,,,,,,,,,,,,,,,,,,
-SC-CoT,gsm8k,2025/1/22,Qwen2-1.5B-Instruct,5.53,86.73,8,"temperature=1, path_num=5",1319,"5,844,218","4,431","3,117,550","2,364","8,961,768",0.0000,,,,,,,,,,,,,,,,,,,,
-SC-CoT,gsm8k,2025/1/22,Qwen2-0.5B-Instruct,3.79,94.84,8,"temperature=1, path_num=5",1319,"6,529,832","4,951","4,003,983","3,036","10,533,815",0.0000,,,,,,,,,,,,,,,,,,,,
+SC-CoT,gsm8k,2025/1/7,gpt-3.5-turbo,79.91,99.92,8,"temperature=1, path_num=5",1319,"2,740,652","2,078","1,348,960","1,023","4,089,612",3.3938,,,,,,,,,,,,,,,,,,,,
+SC-CoT,gsm8k,2025/1/7,Doubao-lite-32k,87.26,99.92,8,"temperature=1, path_num=5",1319,"2,691,714","2,041","1,197,099",908,"3,888,813",0.2083,,,,,,,,,,,,,,,,,,,,
+SC-CoT,gsm8k,2025/1/22,gpt-4o,90.3,99.92,8,"temperature=1, path_num=5",1319,"3,590,336","2,722","2,207,837","1,674","5,798,173",31.0542,,,,,,,,,,,,,,,,,,,,
+SC-CoT,gsm8k,2025/1/22,Qwen2.5-72B-Instruct,93.86,100,8,"temperature=1, path_num=5",1319,"8,136,223","6,168","2,481,785","1,882","10,618,008",5.9858,,,,,,,,,,,,,,,,,,,,
+SC-CoT,gsm8k,2025/1/22,Llama-3.3-70B-Instruct,95.07,100,8,"temperature=1, path_num=5",1319,"8,413,717","6,379","2,585,077","1,960","10,998,794",6.2005,,,,,,,,,,,,,,,,,,,,
+SC-CoT,gsm8k,2025/1/22,Qwen2.5-7B-Instruct,91.13,100,8,"temperature=1, path_num=5",1319,"8,586,888","6,510","2,554,097","1,936","11,140,985",0.0000,,,,,,,,,,,,,,,,,,,,
+SC-CoT,gsm8k,2025/1/22,Llama-3.1-8B-Instruct,73.46,99.55,8,"temperature=1, path_num=5",1319,"8,630,514","6,543","3,148,202","2,387","11,778,716",0.0000,,,,,,,,,,,,,,,,,,,,
+SC-CoT,gsm8k,2025/1/22,Internllm2_5-7B,48.22,98.41,8,"temperature=1, path_num=5",1319,"10,678,792","8,096","3,847,639","2,917","14,526,431",0.0000,,,,,,,,,,,,,,,,,,,,
+SC-CoT,gsm8k,2025/1/22,Qwen2-1.5B-Instruct,11.75,91.89,8,"temperature=1, path_num=5",1319,"9,066,115","6,873","3,345,827","2,537","12,411,942",0.0000,,,,,,,,,,,,,,,,,,,,
+SC-CoT,gsm8k,2025/1/22,Qwen2-0.5B-Instruct,1.67,94.69,8,"temperature=1, path_num=5",1319,"11,019,864","8,355","5,445,856","4,129","16,465,720",0.0000,,,,,,,,,,,,,,,,,,,,
 IO,AQuA,2025/1/7,gpt-3.5-turbo,38.98,100,0,,254,"25,701",101,"16,770",66,"42,471",0.0380,,,,,,,,,,,,,,,,,,,,
 IO,AQuA,2025/1/7,Doubao-lite-32k,79.13,100,0,,254,"33,058",130,"54,684",215,"87,742",0.0058,0.0427（元）,,,,,,,,,,,,,,,,,,,
 IO,AQuA,2025/1/22,gpt-4o,75.59,97.24,0,,254,"25,631",101,"108,121",426,"133,752",1.1453,,,,,,,,,,,,,,,,,,,,
@@ -59,7 +59,7 @@ IO,AQuA,2025/1/22,Llama-3.1-8B-Instruct,51.18,98.82,0,,254,"26,459",104,"106,647
 IO,AQuA,2025/1/22,Internllm2_5-7B,47.64,90.94,0,,254,"50,232",198,"134,809",531,"185,041",0.0000,,,,,,,,,,,,,,,,,,,,
 IO,AQuA,2025/1/22,Qwen2-1.5B-Instruct,29.13,97.64,0,,254,"27,937",110,"43,110",170,"71,047",0.0000,,,,,,,,,,,,,,,,,,,,
 IO,AQuA,2025/1/22,Qwen2-0.5B-Instruct,27.17,98.82,0,,254,"27,937",110,"82,478",325,"110,415",0.0000,,,,,,,,,,,,,,,,,,,,
-CoT,AQuA,2025/1/22,gpt-3.5-turbo,61.02,93.7,0,,254,"25,447",100,"55,346",218,"80,793",0.0957,,,,,,,,,,,,,,,,,,,,
+CoT,AQuA,2025/1/7,gpt-3.5-turbo,61.02,93.7,0,,254,"25,447",100,"55,346",218,"80,793",0.0957,,,,,,,,,,,,,,,,,,,,
 CoT,AQuA,2025/1/7,Doubao-lite-32k,82.68,97.24,0,,254,"27,978",110,"66,599",262,"94,577",0.0066,0.0483 （元）,,,,,,,,,,,,,,,,,,,
 CoT,AQuA,2025/1/22,gpt-4o,82.68,98.03,0,,254,"25,123",99,"97,894",385,"123,017",1.0417,,,,,,,,,,,,,,,,,,,,
 CoT,AQuA,2025/1/22,Qwen2.5-72B-Instruct,86.22,99.21,0,,254,"25,143",99,"118,146",465,"143,289",0.0808,,,,,,,,,,,,,,,,,,,,
@@ -79,16 +79,16 @@ PoT,AQuA,2025/1/22,Llama-3.1-8B-Instruct,36.61,96.85,0,,254,"240,613",947,"50,30
 PoT,AQuA,2025/1/22,Internllm2_5-7B,36.61,98.82,0,,254,"233,505",919,"68,457",270,"301,962",0.0000,,,,,,,,,,,,,,,,,,,,
 PoT,AQuA,2025/1/22,Qwen2-1.5B-Instruct,30.71,96.46,0,,254,"246,560",971,"51,915",204,"298,475",0.0000,,,,,,,,,,,,,,,,,,,,
 PoT,AQuA,2025/1/22,Qwen2-0.5B-Instruct,17.32,92.13,0,,254,"258,867","1,019","63,414",250,"322,281",0.0000,,,,,,,,,,,,,,,,,,,,
-SC-CoT,AQuA,2025/1/7,gpt-3.5-turbo,70.47,98.82,0,"temperature=1, path_num=5",254,"70,157",276,"348,460","1,372","418,617",0.5578,,,,,,,,,,,,,,,,,,,,
-SC-CoT,AQuA,2025/1/7,Doubao-lite-32k,81.5,97.64,0,"temperature=1, path_num=5",254,"83,830",330,"382,016","1,504","465,846",0.0347,,,,,,,,,,,,,,,,,,,,
-SC-CoT,AQuA,2025/1/22,gpt-4o,88.19,100,0,"temperature=1, path_num=5",254,"72,916",287,"605,895","2,385","678,811",6.2412,,,,,,,,,,,,,,,,,,,,
-SC-CoT,AQuA,2025/1/22,Qwen2.5-72B-Instruct,85.83,98.43,0,"temperature=1, path_num=5",254,"241,149",949,"747,909","2,945","989,058",0.5576,,,,,,,,,,,,,,,,,,,,
-SC-CoT,AQuA,2025/1/22,Llama-3.3-70B-Instruct,86.61,99.21,0,"temperature=1, path_num=5",254,"283,248","1,115","753,876","2,968","1,037,124",0.5847,,,,,,,,,,,,,,,,,,,,
-SC-CoT,AQuA,2025/1/22,Qwen2.5-7B-Instruct,81.5,100,0,"temperature=1, path_num=5",254,"278,848","1,098","736,520","2,900","1,015,368",0.0000,,,,,,,,,,,,,,,,,,,,
-SC-CoT,AQuA,2025/1/22,Llama-3.1-8B-Instruct,53.15,96.06,0,"temperature=1, path_num=5",254,"372,968","1,468","668,378","2,631","1,041,346",0.0000,,,,,,,,,,,,,,,,,,,,
-SC-CoT,AQuA,2025/1/22,Internllm2_5-7B,35.85,98.8,0,"temperature=1, path_num=5",254,"530,701","2,089","709,687","2,794","1,240,388",0.0000,,,,,,,,,,,,,,,,,,,,
-SC-CoT,AQuA,2025/1/22,Qwen2-1.5B-Instruct,30.31,97.24,0,"temperature=1, path_num=5",254,"430,703","1,696","726,373","2,860","1,157,076",0.0000,,,,,,,,,,,,,,,,,,,,
-SC-CoT,AQuA,2025/1/22,Qwen2-0.5B-Instruct,30.71,98.43,0,"temperature=1, path_num=5",254,"496,206","1,954","729,333","2,871","1,225,539",0.0000,,,,,,,,,,,,,,,,,,,,
+SC-CoT,AQuA,2025/1/7,gpt-3.5-turbo,66.14,99.21,0,"temperature=1, path_num=5",254,"482,192","1,898","365,143","1,438","847,335",0.7888,,,,,,,,,,,,,,,,,,,,
+SC-CoT,AQuA,2025/1/7,Doubao-lite-32k,81.1,97.24,0,"temperature=1, path_num=5",254,"503,751","1,983","382,235","1,505","885,986",0.0519,,,,,,,,,,,,,,,,,,,,
+SC-CoT,AQuA,2025/1/22,gpt-4o,86.61,98.82,0,"temperature=1, path_num=5",254,"744,478","2,931","628,728","2,475","1,373,206",8.1485,,,,,,,,,,,,,,,,,,,,
+SC-CoT,AQuA,2025/1/22,Qwen2.5-72B-Instruct,85.04,99.21,0,"temperature=1, path_num=5",254,"1,051,218","4,139","784,451","3,088","1,835,669",1.0348,,,,,,,,,,,,,,,,,,,,
+SC-CoT,AQuA,2025/1/22,Llama-3.3-70B-Instruct,82.28,99.21,0,"temperature=1, path_num=5",254,"1,135,251","4,469","772,673","3,042","1,907,924",1.0756,,,,,,,,,,,,,,,,,,,,
+SC-CoT,AQuA,2025/1/22,Qwen2.5-7B-Instruct,79.92,100,0,"temperature=1, path_num=5",254,"1,098,280","4,324","747,052","2,941","1,845,332",0.0000,,,,,,,,,,,,,,,,,,,,
+SC-CoT,AQuA,2025/1/22,Llama-3.1-8B-Instruct,59.45,97.24,0,"temperature=1, path_num=5",254,"971,003","3,823","680,330","2,678","1,651,333",0.0000,,,,,,,,,,,,,,,,,,,,
+SC-CoT,AQuA,2025/1/22,Internllm2_5-7B,39.37,98.03,0,"temperature=1, path_num=5",254,"1,420,494","5,592","875,728","3,448","2,296,222",0.0000,,,,,,,,,,,,,,,,,,,,
+SC-CoT,AQuA,2025/1/22,Qwen2-1.5B-Instruct,23.62,96.46,0,"temperature=1, path_num=5",254,"1,034,362","4,072","740,973","2,917","1,775,335",0.0000,,,,,,,,,,,,,,,,,,,,
+SC-CoT,AQuA,2025/1/22,Qwen2-0.5B-Instruct,22.83,97.24,0,"temperature=1, path_num=5",254,"1,246,929","4,909","968,162","3,812","2,215,091",0.0000,,,,,,,,,,,,,,,,,,,,
 ReAct-Pro*,AQuA,2025/1/7,gpt-3.5-turbo,64.57,98.03,0,max_steps=10,254,"862,614","3,396","40,973",161,"903,587",0.4928,"think-action 单独返回,prompt v1",,,,,,,,,,,,,,,,,,,
 ReAct-Pro*,AQuA,2025/1/7,Doubao-lite-32k,77.56,96.06,0,max_steps=10,254,"977,890","3,850","54,951",216,"1,032,841",0.0445,"think-action 单独返回,prompt v1",,,,,,,,,,,,,,,,,,,
 ReAct-Pro*,AQuA,2025/1/22,gpt-4o,57.48,97.24,0,max_steps=10,254,"615,589","2,424","76,507",301,"692,096",2.3040,"think-action 单独返回,prompt v1",,,,,,,,,,,,,,,,,,,
@@ -98,7 +98,7 @@ ReAct-Pro*,AQuA,2025/1/22,Qwen2.5-7B-Instruct,74.41,99.21,0,max_steps=10,254,"56
 ReAct-Pro*,AQuA,2025/1/22,Llama-3.1-8B-Instruct,55.51,96.85,0,max_steps=10,254,"3,764,723","14,822","576,098","2,268","4,340,821",0.0000,,,,,,,,,,,,,,,,,,,,
 ReAct-Pro*,AQuA,2025/1/22,Internllm2_5-7B,40.94,96.85,0,max_steps=10,254,"3,592,039","14,142","836,762","3,294","4,428,801",0.0000,,,,,,,,,,,,,,,,,,,,
 ReAct-Pro*,AQuA,2025/1/22,Qwen2-1.5B-Instruct,25.59,96.06,0,max_steps=10,254,"4,555,858","17,936","516,146","2,032","5,072,004",0.0000,,,,,,,,,,,,,,,,,,,,
-ReAct-Pro*,AQuA,2025/1/22,Qwen2-0.5B-Instruct,24.02,96.85,0,max_steps=10,254,6344167,"24,977",825920,"3,252","7,170,087",0.0000,,,,,,,,,,,,,,,,,,,,
+ReAct-Pro*,AQuA,2025/1/22,Qwen2-0.5B-Instruct,24.02,96.85,0,max_steps=10,254,"6,344,167","24,977","825,920","3,252","7,170,087",0.0000,,,,,,,,,,,,,,,,,,,,
 ,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
 ,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
 ,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,