Model save

README.md

@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/speechlabx/huggingface/runs/wv9fa7q3) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/speechlabx/huggingface/runs/6hrfnhwk) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.04100553294370786,
-    "train_runtime": 8329.5214,
+    "train_loss": 0.04285794010000496,
+    "train_runtime": 11336.4845,
     "train_samples": 7500,
-    "train_samples_per_second": 0.9,
-    "train_steps_per_second": 0.007
+    "train_samples_per_second": 0.662,
+    "train_steps_per_second": 0.005
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1dc8670668bf49e9b95b5db7836b4028b4a53a65aa498880a009ae4ad34ae1ac
+oid sha256:9185ef8824f31f98137ea8afa0a0be60226fa11498b7aa06a548782154c62da2
 size 4877660776

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1825cfb7168099515cd7dab1c82ecfca113aa9a665683c84039987bd78c5b27d
+oid sha256:65c1f3e4e2659fad3c661a83c78c3a3a5a2b7ed97c0d32db8d1701cf0b34f105
 size 4932751008

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d9c0e7eef7ddb36026815ab0aa3272da0aa29e40635b77567b24e0263ee02c38
+oid sha256:dcc0fed3777fa9e2520d98fb836c49aa5fc2f568041062ee1a729ee225a61093
 size 4330865200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8df3c77817acf9c80b96124a1b0a72964c9662a8ca23c25dc9d697b598ab2b02
+oid sha256:3352f63906588764a7aaf36fd13178ac8c112d8f1127bc82942f4a8e194782fc
 size 1089994880

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.04100553294370786,
-    "train_runtime": 8329.5214,
+    "train_loss": 0.04285794010000496,
+    "train_runtime": 11336.4845,
     "train_samples": 7500,
-    "train_samples_per_second": 0.9,
-    "train_steps_per_second": 0.007
+    "train_samples_per_second": 0.662,
+    "train_steps_per_second": 0.005
 }

trainer_state.json

@@ -2,7 +2,7 @@
   "best_metric": null,
   "best_model_checkpoint": null,
   "epoch": 0.9893390191897654,
-  "eval_steps": 100,
+  "eval_steps": 10000,
   "global_step": 58,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
@@ -10,824 +10,839 @@
   "log_history": [
     {
       "clip_ratio": 0.0,
-      "completion_length": 605.9933242797852,
+      "completion_length": 604.5960006713867,
       "epoch": 0.017057569296375266,
-      "grad_norm": 0.46656012535095215,
+      "grad_norm": 0.4511799216270447,
       "kl": 0.0,
       "learning_rate": 5e-07,
-      "loss": 0.0328,
-      "reward": 0.6272321715950966,
-      "reward_std": 0.3389472607523203,
-      "rewards/accuracy_reward": 0.6272321715950966,
+      "loss": 0.0272,
+      "reward": 0.6261160969734192,
+      "reward_std": 0.3447213862091303,
+      "rewards/accuracy_reward": 0.6261160969734192,
       "rewards/format_reward": 0.0,
       "step": 1
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 595.8381958007812,
+      "completion_length": 602.7522583007812,
       "epoch": 0.03411513859275053,
-      "grad_norm": 0.40490883588790894,
+      "grad_norm": 0.3016141951084137,
       "kl": 0.0,
       "learning_rate": 1e-06,
-      "loss": 0.029,
-      "reward": 0.5859375260770321,
-      "reward_std": 0.37348853051662445,
-      "rewards/accuracy_reward": 0.5859375260770321,
+      "loss": 0.0351,
+      "reward": 0.6026785969734192,
+      "reward_std": 0.35665313154459,
+      "rewards/accuracy_reward": 0.6026785969734192,
       "rewards/format_reward": 0.0,
       "step": 2
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 616.5044860839844,
+      "completion_length": 631.8515930175781,
       "epoch": 0.0511727078891258,
-      "grad_norm": 0.5747449398040771,
-      "kl": 0.00019681453704833984,
+      "grad_norm": 0.3726074695587158,
+      "kl": 0.00022530555725097656,
       "learning_rate": 1.5e-06,
-      "loss": 0.0331,
-      "reward": 0.6138393133878708,
-      "reward_std": 0.34731038846075535,
-      "rewards/accuracy_reward": 0.6138393133878708,
+      "loss": 0.0455,
+      "reward": 0.6149553880095482,
+      "reward_std": 0.35506617091596127,
+      "rewards/accuracy_reward": 0.6149553880095482,
       "rewards/format_reward": 0.0,
       "step": 3
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 593.1674423217773,
+      "completion_length": 596.8404312133789,
       "epoch": 0.06823027718550106,
-      "grad_norm": 0.37437704205513,
-      "kl": 0.00048792362213134766,
+      "grad_norm": 0.7917911410331726,
+      "kl": 0.0002683401107788086,
       "learning_rate": 2e-06,
-      "loss": -0.003,
-      "reward": 0.5725446753203869,
-      "reward_std": 0.35837166383862495,
-      "rewards/accuracy_reward": 0.5725446753203869,
+      "loss": 0.0196,
+      "reward": 0.564732164144516,
+      "reward_std": 0.35563987866044044,
+      "rewards/accuracy_reward": 0.564732164144516,
       "rewards/format_reward": 0.0,
       "step": 4
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 595.5591735839844,
+      "completion_length": 590.5893135070801,
       "epoch": 0.08528784648187633,
-      "grad_norm": 0.4537689983844757,
-      "kl": 0.000583648681640625,
+      "grad_norm": 0.5213038325309753,
+      "kl": 0.0002919435501098633,
       "learning_rate": 2.5e-06,
-      "loss": 0.038,
-      "reward": 0.604910746216774,
-      "reward_std": 0.3501625321805477,
-      "rewards/accuracy_reward": 0.604910746216774,
+      "loss": 0.007,
+      "reward": 0.640625037252903,
+      "reward_std": 0.3181990496814251,
+      "rewards/accuracy_reward": 0.640625037252903,
       "rewards/format_reward": 0.0,
       "step": 5
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 601.1138725280762,
+      "completion_length": 586.8504676818848,
       "epoch": 0.1023454157782516,
-      "grad_norm": 2.8121519088745117,
-      "kl": 0.002495288848876953,
+      "grad_norm": 0.333626925945282,
+      "kl": 0.0005509853363037109,
       "learning_rate": 3e-06,
-      "loss": 0.051,
-      "reward": 0.6339286044239998,
-      "reward_std": 0.28945007361471653,
-      "rewards/accuracy_reward": 0.632812537252903,
-      "rewards/format_reward": 0.0011160714784637094,
+      "loss": 0.03,
+      "reward": 0.6406250298023224,
+      "reward_std": 0.3159851096570492,
+      "rewards/accuracy_reward": 0.6406250298023224,
+      "rewards/format_reward": 0.0,
       "step": 6
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 619.041316986084,
+      "completion_length": 611.2210083007812,
       "epoch": 0.11940298507462686,
-      "grad_norm": 0.6353834867477417,
-      "kl": 0.0027112960815429688,
+      "grad_norm": 0.33720001578330994,
+      "kl": 0.0010228157043457031,
       "learning_rate": 2.9972633313349763e-06,
-      "loss": 0.0694,
-      "reward": 0.6618303805589676,
-      "reward_std": 0.30959521792829037,
-      "rewards/accuracy_reward": 0.6618303805589676,
+      "loss": 0.0505,
+      "reward": 0.6774553880095482,
+      "reward_std": 0.31844254955649376,
+      "rewards/accuracy_reward": 0.6774553880095482,
       "rewards/format_reward": 0.0,
       "step": 7
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 630.1774749755859,
+      "completion_length": 633.0658798217773,
       "epoch": 0.13646055437100213,
-      "grad_norm": 0.5084631443023682,
-      "kl": 0.003963470458984375,
+      "grad_norm": 40.491729736328125,
+      "kl": 0.023244857788085938,
       "learning_rate": 2.989063311147081e-06,
-      "loss": 0.0362,
-      "reward": 0.6584821715950966,
-      "reward_std": 0.31060847640037537,
-      "rewards/accuracy_reward": 0.6584821715950966,
+      "loss": 0.0587,
+      "reward": 0.6495535969734192,
+      "reward_std": 0.30738127417862415,
+      "rewards/accuracy_reward": 0.6495535969734192,
       "rewards/format_reward": 0.0,
       "step": 8
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 605.6975860595703,
+      "completion_length": 596.904052734375,
       "epoch": 0.1535181236673774,
-      "grad_norm": 0.3929789066314697,
-      "kl": 0.007457733154296875,
+      "grad_norm": 191.003173828125,
+      "kl": 0.11961555480957031,
       "learning_rate": 2.9754298604207156e-06,
-      "loss": 0.0784,
-      "reward": 0.7433036044239998,
-      "reward_std": 0.28443033434450626,
-      "rewards/accuracy_reward": 0.7433036044239998,
+      "loss": 0.0912,
+      "reward": 0.7299107387661934,
+      "reward_std": 0.3143673036247492,
+      "rewards/accuracy_reward": 0.7299107387661934,
       "rewards/format_reward": 0.0,
       "step": 9
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 614.2846221923828,
+      "completion_length": 620.3705596923828,
       "epoch": 0.17057569296375266,
-      "grad_norm": 0.46036890149116516,
-      "kl": 0.020414352416992188,
+      "grad_norm": 0.40953710675239563,
+      "kl": 0.0033578872680664062,
       "learning_rate": 2.956412726139078e-06,
-      "loss": 0.0579,
-      "reward": 0.7031250223517418,
-      "reward_std": 0.26031214371323586,
-      "rewards/accuracy_reward": 0.7031250223517418,
+      "loss": 0.083,
+      "reward": 0.7209821864962578,
+      "reward_std": 0.2721347473561764,
+      "rewards/accuracy_reward": 0.7209821864962578,
       "rewards/format_reward": 0.0,
       "step": 10
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 648.0703506469727,
+      "completion_length": 652.9877548217773,
       "epoch": 0.18763326226012794,
-      "grad_norm": 0.40724363923072815,
-      "kl": 0.003391265869140625,
+      "grad_norm": 0.19379274547100067,
+      "kl": 0.003864288330078125,
       "learning_rate": 2.9320812997628183e-06,
-      "loss": 0.057,
-      "reward": 0.729910746216774,
-      "reward_std": 0.23080609552562237,
-      "rewards/accuracy_reward": 0.729910746216774,
+      "loss": 0.0712,
+      "reward": 0.7087053880095482,
+      "reward_std": 0.26535856537520885,
+      "rewards/accuracy_reward": 0.7087053880095482,
       "rewards/format_reward": 0.0,
       "step": 11
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 572.5647506713867,
+      "completion_length": 577.8091888427734,
       "epoch": 0.2046908315565032,
-      "grad_norm": 0.3498363792896271,
-      "kl": 0.0052776336669921875,
+      "grad_norm": 0.7543983459472656,
+      "kl": 0.0048828125,
       "learning_rate": 2.9025243640281224e-06,
-      "loss": 0.0529,
-      "reward": 0.8069196864962578,
-      "reward_std": 0.22050337865948677,
-      "rewards/accuracy_reward": 0.8069196864962578,
+      "loss": 0.0758,
+      "reward": 0.7946428954601288,
+      "reward_std": 0.19575820118188858,
+      "rewards/accuracy_reward": 0.7946428954601288,
       "rewards/format_reward": 0.0,
       "step": 12
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 588.2254791259766,
+      "completion_length": 605.303596496582,
       "epoch": 0.22174840085287847,
-      "grad_norm": 0.5149727463722229,
-      "kl": 0.0054779052734375,
+      "grad_norm": 1.7850338220596313,
+      "kl": 0.005886077880859375,
       "learning_rate": 2.8678497689881355e-06,
-      "loss": 0.0505,
-      "reward": 0.7767857536673546,
-      "reward_std": 0.1721354192122817,
-      "rewards/accuracy_reward": 0.7767857536673546,
+      "loss": 0.062,
+      "reward": 0.7589286118745804,
+      "reward_std": 0.2227060990408063,
+      "rewards/accuracy_reward": 0.7589286118745804,
       "rewards/format_reward": 0.0,
       "step": 13
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 619.4620895385742,
+      "completion_length": 609.9096221923828,
       "epoch": 0.23880597014925373,
-      "grad_norm": 0.18762321770191193,
-      "kl": 0.004208564758300781,
+      "grad_norm": 0.2924557030200958,
+      "kl": 0.0028257369995117188,
       "learning_rate": 2.8281840384798147e-06,
-      "loss": 0.0662,
-      "reward": 0.7488839626312256,
-      "reward_std": 0.20387782901525497,
-      "rewards/accuracy_reward": 0.7488839626312256,
+      "loss": 0.0644,
+      "reward": 0.7366071715950966,
+      "reward_std": 0.20327048748731613,
+      "rewards/accuracy_reward": 0.7366071715950966,
       "rewards/format_reward": 0.0,
       "step": 14
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 595.3817291259766,
+      "completion_length": 607.3292617797852,
       "epoch": 0.255863539445629,
-      "grad_norm": 0.641958475112915,
-      "kl": 0.004180908203125,
+      "grad_norm": 0.19465771317481995,
+      "kl": 0.0043182373046875,
       "learning_rate": 2.7836719084521715e-06,
-      "loss": 0.0681,
-      "reward": 0.7667411118745804,
-      "reward_std": 0.2224541725590825,
-      "rewards/accuracy_reward": 0.7667411118745804,
+      "loss": 0.0529,
+      "reward": 0.773437537252903,
+      "reward_std": 0.2134528523311019,
+      "rewards/accuracy_reward": 0.773437537252903,
       "rewards/format_reward": 0.0,
       "step": 15
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 583.5859680175781,
+      "completion_length": 582.2745819091797,
       "epoch": 0.27292110874200426,
-      "grad_norm": 0.741455078125,
-      "kl": 0.0067882537841796875,
+      "grad_norm": 0.19785651564598083,
+      "kl": 0.00447845458984375,
       "learning_rate": 2.7344757988404844e-06,
-      "loss": 0.0656,
-      "reward": 0.7901786118745804,
-      "reward_std": 0.21803472563624382,
-      "rewards/accuracy_reward": 0.7901786118745804,
+      "loss": 0.0635,
+      "reward": 0.792410746216774,
+      "reward_std": 0.2150930892676115,
+      "rewards/accuracy_reward": 0.792410746216774,
       "rewards/format_reward": 0.0,
       "step": 16
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 597.1685638427734,
+      "completion_length": 607.478816986084,
       "epoch": 0.2899786780383795,
-      "grad_norm": 0.23518237471580505,
-      "kl": 0.003398895263671875,
+      "grad_norm": 2.073848009109497,
+      "kl": 0.012720108032226562,
       "learning_rate": 2.680775220913575e-06,
-      "loss": 0.0435,
-      "reward": 0.7645089700818062,
-      "reward_std": 0.17900951812043786,
-      "rewards/accuracy_reward": 0.7645089700818062,
+      "loss": 0.0424,
+      "reward": 0.756696455180645,
+      "reward_std": 0.1880332687869668,
+      "rewards/accuracy_reward": 0.756696455180645,
       "rewards/format_reward": 0.0,
       "step": 17
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 612.630615234375,
+      "completion_length": 610.4665374755859,
       "epoch": 0.3070362473347548,
-      "grad_norm": 0.27309614419937134,
-      "kl": 0.0040035247802734375,
+      "grad_norm": 0.23339830338954926,
+      "kl": 0.01509857177734375,
       "learning_rate": 2.6227661222566517e-06,
-      "loss": 0.055,
-      "reward": 0.7399553954601288,
-      "reward_std": 0.2160194292664528,
-      "rewards/accuracy_reward": 0.7399553954601288,
+      "loss": 0.0432,
+      "reward": 0.7343750447034836,
+      "reward_std": 0.21946769207715988,
+      "rewards/accuracy_reward": 0.7343750447034836,
       "rewards/format_reward": 0.0,
       "step": 18
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 596.2109603881836,
+      "completion_length": 595.0357475280762,
       "epoch": 0.32409381663113007,
-      "grad_norm": 0.1339673399925232,
-      "kl": 0.0028057098388671875,
+      "grad_norm": 0.10285016894340515,
+      "kl": 0.0044097900390625,
       "learning_rate": 2.5606601717798212e-06,
-      "loss": 0.0425,
-      "reward": 0.7700893059372902,
-      "reward_std": 0.19220922887325287,
-      "rewards/accuracy_reward": 0.7700893059372902,
+      "loss": 0.0378,
+      "reward": 0.7857143208384514,
+      "reward_std": 0.17597838956862688,
+      "rewards/accuracy_reward": 0.7857143208384514,
       "rewards/format_reward": 0.0,
       "step": 19
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 552.1049461364746,
+      "completion_length": 565.811408996582,
       "epoch": 0.3411513859275053,
-      "grad_norm": 0.4360298216342926,
-      "kl": 0.0038690567016601562,
+      "grad_norm": 0.14462673664093018,
+      "kl": 0.003070831298828125,
       "learning_rate": 2.4946839873611927e-06,
-      "loss": 0.0449,
-      "reward": 0.7912946790456772,
-      "reward_std": 0.19673781003803015,
-      "rewards/accuracy_reward": 0.7912946790456772,
+      "loss": 0.051,
+      "reward": 0.762276828289032,
+      "reward_std": 0.21532537415623665,
+      "rewards/accuracy_reward": 0.762276828289032,
       "rewards/format_reward": 0.0,
       "step": 20
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 606.3794937133789,
+      "completion_length": 620.9185562133789,
       "epoch": 0.3582089552238806,
-      "grad_norm": 0.27116331458091736,
-      "kl": 0.0029468536376953125,
+      "grad_norm": 0.6581376791000366,
+      "kl": 0.00328826904296875,
       "learning_rate": 2.425078308942815e-06,
-      "loss": 0.0265,
-      "reward": 0.7734375298023224,
-      "reward_std": 0.17621066141873598,
-      "rewards/accuracy_reward": 0.7734375298023224,
+      "loss": 0.0453,
+      "reward": 0.757812537252903,
+      "reward_std": 0.2050897255539894,
+      "rewards/accuracy_reward": 0.757812537252903,
       "rewards/format_reward": 0.0,
       "step": 21
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 585.0826187133789,
+      "completion_length": 590.5056076049805,
       "epoch": 0.3752665245202559,
-      "grad_norm": 0.15653851628303528,
-      "kl": 0.0028667449951171875,
+      "grad_norm": 1.3106849193572998,
+      "kl": 0.0048999786376953125,
       "learning_rate": 2.3520971200967337e-06,
-      "loss": 0.0473,
-      "reward": 0.8035714700818062,
-      "reward_std": 0.16157799493521452,
-      "rewards/accuracy_reward": 0.8035714700818062,
+      "loss": 0.0458,
+      "reward": 0.791294664144516,
+      "reward_std": 0.20316135231405497,
+      "rewards/accuracy_reward": 0.791294664144516,
       "rewards/format_reward": 0.0,
       "step": 22
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 596.0055999755859,
+      "completion_length": 595.1216735839844,
       "epoch": 0.39232409381663114,
-      "grad_norm": 0.1592758744955063,
-      "kl": 0.0041027069091796875,
+      "grad_norm": 0.15704886615276337,
+      "kl": 0.00351715087890625,
       "learning_rate": 2.276006721266485e-06,
-      "loss": 0.0449,
-      "reward": 0.7779018208384514,
-      "reward_std": 0.1884950390085578,
-      "rewards/accuracy_reward": 0.7779018208384514,
+      "loss": 0.0422,
+      "reward": 0.7656250298023224,
+      "reward_std": 0.19919524434953928,
+      "rewards/accuracy_reward": 0.7656250298023224,
       "rewards/format_reward": 0.0,
       "step": 23
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 592.358283996582,
+      "completion_length": 597.1718978881836,
       "epoch": 0.4093816631130064,
-      "grad_norm": 0.7523362040519714,
-      "kl": 0.0069179534912109375,
+      "grad_norm": 0.12948359549045563,
+      "kl": 0.004302978515625,
       "learning_rate": 2.1970847580656528e-06,
-      "loss": 0.0609,
-      "reward": 0.7511161044239998,
-      "reward_std": 0.19366461411118507,
-      "rewards/accuracy_reward": 0.7511161044239998,
+      "loss": 0.0466,
+      "reward": 0.7522321790456772,
+      "reward_std": 0.18040904868394136,
+      "rewards/accuracy_reward": 0.7522321790456772,
       "rewards/format_reward": 0.0,
       "step": 24
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 584.0781593322754,
+      "completion_length": 601.4386405944824,
       "epoch": 0.42643923240938164,
-      "grad_norm": 0.1614408791065216,
-      "kl": 0.0040912628173828125,
+      "grad_norm": 0.1379498988389969,
+      "kl": 0.0035552978515625,
       "learning_rate": 2.1156192081791355e-06,
-      "loss": 0.0412,
-      "reward": 0.731026828289032,
-      "reward_std": 0.19054656103253365,
-      "rewards/accuracy_reward": 0.731026828289032,
+      "loss": 0.0661,
+      "reward": 0.718750037252903,
+      "reward_std": 0.2221574056893587,
+      "rewards/accuracy_reward": 0.718750037252903,
       "rewards/format_reward": 0.0,
       "step": 25
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 586.0480117797852,
+      "completion_length": 590.9721260070801,
       "epoch": 0.44349680170575695,
-      "grad_norm": 0.14655889570713043,
-      "kl": 0.0051975250244140625,
+      "grad_norm": 0.6328846216201782,
+      "kl": 0.003810882568359375,
       "learning_rate": 2.0319073305638034e-06,
-      "loss": 0.0398,
+      "loss": 0.0435,
       "reward": 0.753348246216774,
-      "reward_std": 0.19846755359321833,
+      "reward_std": 0.19143945816904306,
       "rewards/accuracy_reward": 0.753348246216774,
       "rewards/format_reward": 0.0,
       "step": 26
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 583.4933319091797,
+      "completion_length": 588.3326263427734,
       "epoch": 0.4605543710021322,
-      "grad_norm": 0.2977466285228729,
-      "kl": 0.0040130615234375,
+      "grad_norm": 0.1565869301557541,
+      "kl": 0.0039806365966796875,
       "learning_rate": 1.9462545807828044e-06,
-      "loss": 0.0443,
-      "reward": 0.761160746216774,
-      "reward_std": 0.17427107319235802,
-      "rewards/accuracy_reward": 0.761160746216774,
+      "loss": 0.05,
+      "reward": 0.7354911044239998,
+      "reward_std": 0.1781614711508155,
+      "rewards/accuracy_reward": 0.7354911044239998,
       "rewards/format_reward": 0.0,
       "step": 27
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 565.6919822692871,
+      "completion_length": 583.0100708007812,
       "epoch": 0.47761194029850745,
-      "grad_norm": 0.3453241288661957,
-      "kl": 0.0048007965087890625,
+      "grad_norm": 0.5358207821846008,
+      "kl": 0.013216018676757812,
       "learning_rate": 1.8589734964313368e-06,
-      "loss": 0.0312,
-      "reward": 0.7801339700818062,
-      "reward_std": 0.17806075559929013,
-      "rewards/accuracy_reward": 0.7801339700818062,
+      "loss": 0.0499,
+      "reward": 0.7957589700818062,
+      "reward_std": 0.17203470412641764,
+      "rewards/accuracy_reward": 0.7957589700818062,
       "rewards/format_reward": 0.0,
       "step": 28
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 608.4219055175781,
+      "completion_length": 617.6105194091797,
       "epoch": 0.4946695095948827,
-      "grad_norm": 0.34829476475715637,
-      "kl": 0.0038909912109375,
+      "grad_norm": 0.10866988450288773,
+      "kl": 0.0028591156005859375,
       "learning_rate": 1.7703825567208588e-06,
-      "loss": 0.0357,
-      "reward": 0.7455357536673546,
-      "reward_std": 0.1893458729609847,
-      "rewards/accuracy_reward": 0.7455357536673546,
+      "loss": 0.0461,
+      "reward": 0.7343750298023224,
+      "reward_std": 0.1905689835548401,
+      "rewards/accuracy_reward": 0.7343750298023224,
       "rewards/format_reward": 0.0,
       "step": 29
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 638.1015930175781,
+      "completion_length": 628.3661041259766,
       "epoch": 0.511727078891258,
-      "grad_norm": 0.8564308881759644,
-      "kl": 0.0047130584716796875,
+      "grad_norm": 0.1341499537229538,
+      "kl": 0.003131866455078125,
       "learning_rate": 1.6808050203829845e-06,
-      "loss": 0.0605,
-      "reward": 0.7120536044239998,
-      "reward_std": 0.1940593123435974,
-      "rewards/accuracy_reward": 0.7120536044239998,
+      "loss": 0.0238,
+      "reward": 0.735491119325161,
+      "reward_std": 0.17904315516352654,
+      "rewards/accuracy_reward": 0.735491119325161,
       "rewards/format_reward": 0.0,
       "step": 30
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 595.8727874755859,
+      "completion_length": 589.1886444091797,
       "epoch": 0.5287846481876333,
-      "grad_norm": 0.4144252836704254,
-      "kl": 0.0044384002685546875,
+      "grad_norm": 0.12772534787654877,
+      "kl": 0.004833221435546875,
       "learning_rate": 1.5905677461334292e-06,
-      "loss": 0.0462,
-      "reward": 0.7343750298023224,
-      "reward_std": 0.19265135563910007,
-      "rewards/accuracy_reward": 0.7343750298023224,
+      "loss": 0.0384,
+      "reward": 0.7243303880095482,
+      "reward_std": 0.19990051537752151,
+      "rewards/accuracy_reward": 0.7243303880095482,
       "rewards/format_reward": 0.0,
       "step": 31
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 583.022331237793,
+      "completion_length": 577.6808319091797,
       "epoch": 0.5458422174840085,
-      "grad_norm": 0.2915865480899811,
-      "kl": 0.0038242340087890625,
+      "grad_norm": 0.1517443060874939,
+      "kl": 0.0039844512939453125,
       "learning_rate": 1.5e-06,
-      "loss": 0.0366,
-      "reward": 0.7656250298023224,
-      "reward_std": 0.1911288918927312,
-      "rewards/accuracy_reward": 0.7656250298023224,
+      "loss": 0.0365,
+      "reward": 0.7633928954601288,
+      "reward_std": 0.18407837813720107,
+      "rewards/accuracy_reward": 0.7633928954601288,
       "rewards/format_reward": 0.0,
       "step": 32
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 597.7399787902832,
+      "completion_length": 587.8839569091797,
       "epoch": 0.5628997867803838,
-      "grad_norm": 0.12201324105262756,
-      "kl": 0.0044403076171875,
+      "grad_norm": 0.1479600965976715,
+      "kl": 0.0033779144287109375,
       "learning_rate": 1.4094322538665708e-06,
-      "loss": 0.0375,
-      "reward": 0.7366071790456772,
-      "reward_std": 0.16133728623390198,
-      "rewards/accuracy_reward": 0.7366071790456772,
+      "loss": 0.0441,
+      "reward": 0.7466518208384514,
+      "reward_std": 0.19517328590154648,
+      "rewards/accuracy_reward": 0.7466518208384514,
       "rewards/format_reward": 0.0,
       "step": 33
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 589.4151954650879,
+      "completion_length": 591.8169937133789,
       "epoch": 0.579957356076759,
-      "grad_norm": 0.16991916298866272,
-      "kl": 0.0041408538818359375,
+      "grad_norm": 0.1458793729543686,
+      "kl": 0.0035686492919921875,
       "learning_rate": 1.3191949796170155e-06,
-      "loss": 0.0318,
-      "reward": 0.7321428954601288,
-      "reward_std": 0.16871801298111677,
-      "rewards/accuracy_reward": 0.7321428954601288,
+      "loss": 0.0396,
+      "reward": 0.7232143059372902,
+      "reward_std": 0.19442316610366106,
+      "rewards/accuracy_reward": 0.7232143059372902,
       "rewards/format_reward": 0.0,
       "step": 34
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 625.5457954406738,
+      "completion_length": 629.7857322692871,
       "epoch": 0.5970149253731343,
-      "grad_norm": 0.18359360098838806,
-      "kl": 0.0037403106689453125,
+      "grad_norm": 0.09501513838768005,
+      "kl": 0.008701324462890625,
       "learning_rate": 1.2296174432791415e-06,
-      "loss": 0.0318,
-      "reward": 0.7488839775323868,
-      "reward_std": 0.16130364406853914,
-      "rewards/accuracy_reward": 0.7488839775323868,
+      "loss": 0.0273,
+      "reward": 0.7321428954601288,
+      "reward_std": 0.19501928705722094,
+      "rewards/accuracy_reward": 0.7321428954601288,
       "rewards/format_reward": 0.0,
       "step": 35
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 558.873908996582,
+      "completion_length": 568.5536155700684,
       "epoch": 0.6140724946695096,
-      "grad_norm": 0.16755032539367676,
-      "kl": 0.0041866302490234375,
+      "grad_norm": 0.1745123714208603,
+      "kl": 0.0050048828125,
       "learning_rate": 1.141026503568664e-06,
-      "loss": 0.0348,
-      "reward": 0.750000037252903,
-      "reward_std": 0.15614807326346636,
-      "rewards/accuracy_reward": 0.750000037252903,
+      "loss": 0.0448,
+      "reward": 0.7600446715950966,
+      "reward_std": 0.18900445476174355,
+      "rewards/accuracy_reward": 0.7600446715950966,
       "rewards/format_reward": 0.0,
       "step": 36
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 577.9743461608887,
+      "completion_length": 580.4486923217773,
       "epoch": 0.6311300639658849,
-      "grad_norm": 0.22190368175506592,
-      "kl": 0.0046405792236328125,
+      "grad_norm": 0.16805370151996613,
+      "kl": 0.0038299560546875,
       "learning_rate": 1.0537454192171958e-06,
-      "loss": 0.0402,
-      "reward": 0.8002232536673546,
-      "reward_std": 0.17436057049781084,
-      "rewards/accuracy_reward": 0.8002232536673546,
+      "loss": 0.0328,
+      "reward": 0.797991119325161,
+      "reward_std": 0.1671001985669136,
+      "rewards/accuracy_reward": 0.797991119325161,
       "rewards/format_reward": 0.0,
       "step": 37
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 581.0558319091797,
+      "completion_length": 574.9922180175781,
       "epoch": 0.6481876332622601,
-      "grad_norm": 0.1611102819442749,
-      "kl": 0.00406646728515625,
+      "grad_norm": 0.1520647555589676,
+      "kl": 0.00421142578125,
       "learning_rate": 9.680926694361964e-07,
-      "loss": 0.0365,
-      "reward": 0.7678571715950966,
-      "reward_std": 0.15295173227787018,
-      "rewards/accuracy_reward": 0.7678571715950966,
+      "loss": 0.0415,
+      "reward": 0.7511161118745804,
+      "reward_std": 0.18564290553331375,
+      "rewards/accuracy_reward": 0.7511161118745804,
       "rewards/format_reward": 0.0,
       "step": 38
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 554.3147659301758,
+      "completion_length": 555.4922180175781,
       "epoch": 0.6652452025586354,
-      "grad_norm": 0.6986324787139893,
-      "kl": 0.0059719085693359375,
+      "grad_norm": 0.1991657167673111,
+      "kl": 0.004291534423828125,
       "learning_rate": 8.843807918208651e-07,
-      "loss": 0.0387,
-      "reward": 0.801339328289032,
-      "reward_std": 0.16045559756457806,
-      "rewards/accuracy_reward": 0.801339328289032,
+      "loss": 0.0422,
+      "reward": 0.7823661118745804,
+      "reward_std": 0.17239855788648129,
+      "rewards/accuracy_reward": 0.7823661118745804,
       "rewards/format_reward": 0.0,
       "step": 39
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 628.6562652587891,
+      "completion_length": 625.3661117553711,
       "epoch": 0.6823027718550106,
-      "grad_norm": 0.2882983386516571,
-      "kl": 0.0050792694091796875,
+      "grad_norm": 0.11269740760326385,
+      "kl": 0.0033588409423828125,
       "learning_rate": 8.029152419343472e-07,
-      "loss": 0.0423,
-      "reward": 0.7388393208384514,
-      "reward_std": 0.18197357282042503,
-      "rewards/accuracy_reward": 0.7388393208384514,
+      "loss": 0.0409,
+      "reward": 0.7075893133878708,
+      "reward_std": 0.20507850963622332,
+      "rewards/accuracy_reward": 0.7075893133878708,
       "rewards/format_reward": 0.0,
       "step": 40
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 604.989990234375,
+      "completion_length": 610.1897506713867,
       "epoch": 0.6993603411513859,
-      "grad_norm": 0.24079355597496033,
-      "kl": 0.004535675048828125,
+      "grad_norm": 0.21570497751235962,
+      "kl": 0.005218505859375,
       "learning_rate": 7.239932787335147e-07,
-      "loss": 0.0428,
-      "reward": 0.746651828289032,
-      "reward_std": 0.21638327650725842,
-      "rewards/accuracy_reward": 0.746651828289032,
+      "loss": 0.0379,
+      "reward": 0.7656250298023224,
+      "reward_std": 0.19328955188393593,
+      "rewards/accuracy_reward": 0.7656250298023224,
       "rewards/format_reward": 0.0,
       "step": 41
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 588.8884124755859,
+      "completion_length": 601.0435485839844,
       "epoch": 0.7164179104477612,
-      "grad_norm": 0.1857738345861435,
-      "kl": 0.0051898956298828125,
+      "grad_norm": 0.17806975543498993,
+      "kl": 0.003978729248046875,
       "learning_rate": 6.479028799032664e-07,
-      "loss": 0.0348,
+      "loss": 0.0424,
       "reward": 0.7667410969734192,
-      "reward_std": 0.1482915743254125,
+      "reward_std": 0.2007934171706438,
       "rewards/accuracy_reward": 0.7667410969734192,
       "rewards/format_reward": 0.0,
       "step": 42
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 633.1930999755859,
+      "completion_length": 634.1518173217773,
       "epoch": 0.7334754797441365,
-      "grad_norm": 0.12053947895765305,
-      "kl": 0.004100799560546875,
+      "grad_norm": 0.1565958708524704,
+      "kl": 0.005039215087890625,
       "learning_rate": 5.749216910571854e-07,
-      "loss": 0.0326,
-      "reward": 0.7578125298023224,
-      "reward_std": 0.18650216422975063,
-      "rewards/accuracy_reward": 0.7578125298023224,
+      "loss": 0.0482,
+      "reward": 0.7287946790456772,
+      "reward_std": 0.22933948785066605,
+      "rewards/accuracy_reward": 0.7287946790456772,
       "rewards/format_reward": 0.0,
       "step": 43
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 591.989990234375,
+      "completion_length": 586.2723388671875,
       "epoch": 0.7505330490405118,
-      "grad_norm": 0.2785487473011017,
-      "kl": 0.0066280364990234375,
+      "grad_norm": 0.13228431344032288,
+      "kl": 0.0039825439453125,
       "learning_rate": 5.053160126388076e-07,
-      "loss": 0.0232,
-      "reward": 0.7243303880095482,
-      "reward_std": 0.16931413393467665,
-      "rewards/accuracy_reward": 0.7243303880095482,
+      "loss": 0.0416,
+      "reward": 0.7366071864962578,
+      "reward_std": 0.1808932526037097,
+      "rewards/accuracy_reward": 0.7366071864962578,
       "rewards/format_reward": 0.0,
       "step": 44
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 564.8236808776855,
+      "completion_length": 576.3806037902832,
       "epoch": 0.767590618336887,
-      "grad_norm": 0.20789861679077148,
-      "kl": 0.00405120849609375,
+      "grad_norm": 0.13988560438156128,
+      "kl": 0.0044498443603515625,
       "learning_rate": 4.3933982822017883e-07,
-      "loss": 0.0318,
-      "reward": 0.7935268133878708,
-      "reward_std": 0.15123321348801255,
-      "rewards/accuracy_reward": 0.7935268133878708,
+      "loss": 0.0219,
+      "reward": 0.7633928954601288,
+      "reward_std": 0.17312624771147966,
+      "rewards/accuracy_reward": 0.7633928954601288,
       "rewards/format_reward": 0.0,
       "step": 45
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 609.4509048461914,
+      "completion_length": 604.428596496582,
       "epoch": 0.7846481876332623,
-      "grad_norm": 0.5048084855079651,
-      "kl": 0.0051288604736328125,
+      "grad_norm": 0.10653964430093765,
+      "kl": 0.0032711029052734375,
       "learning_rate": 3.772338777433482e-07,
-      "loss": 0.0484,
-      "reward": 0.7388393133878708,
-      "reward_std": 0.19954787380993366,
-      "rewards/accuracy_reward": 0.7388393133878708,
+      "loss": 0.0434,
+      "reward": 0.7220982536673546,
+      "reward_std": 0.2057167086750269,
+      "rewards/accuracy_reward": 0.7220982536673546,
       "rewards/format_reward": 0.0,
       "step": 46
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 591.3649749755859,
+      "completion_length": 587.4810485839844,
       "epoch": 0.8017057569296375,
-      "grad_norm": 0.1130928322672844,
-      "kl": 0.00443267822265625,
+      "grad_norm": 0.12146904319524765,
+      "kl": 0.0043125152587890625,
       "learning_rate": 3.192247790864249e-07,
-      "loss": 0.0368,
-      "reward": 0.7533482536673546,
-      "reward_std": 0.20039592683315277,
-      "rewards/accuracy_reward": 0.7533482536673546,
+      "loss": 0.0364,
+      "reward": 0.718750037252903,
+      "reward_std": 0.21366270910948515,
+      "rewards/accuracy_reward": 0.718750037252903,
       "rewards/format_reward": 0.0,
       "step": 47
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 579.0134162902832,
+      "completion_length": 578.7477912902832,
       "epoch": 0.8187633262260128,
-      "grad_norm": 0.14524802565574646,
-      "kl": 0.0044956207275390625,
+      "grad_norm": 0.38011038303375244,
+      "kl": 0.005626678466796875,
       "learning_rate": 2.6552420115951547e-07,
-      "loss": 0.0361,
-      "reward": 0.7845982536673546,
-      "reward_std": 0.1610517231747508,
-      "rewards/accuracy_reward": 0.7845982536673546,
+      "loss": 0.0458,
+      "reward": 0.768973246216774,
+      "reward_std": 0.18335067853331566,
+      "rewards/accuracy_reward": 0.768973246216774,
       "rewards/format_reward": 0.0,
       "step": 48
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 614.7578506469727,
+      "completion_length": 619.3940048217773,
       "epoch": 0.835820895522388,
-      "grad_norm": 0.1255880445241928,
-      "kl": 0.00421142578125,
+      "grad_norm": 0.12710346281528473,
+      "kl": 0.008390426635742188,
       "learning_rate": 2.163280915478289e-07,
-      "loss": 0.0287,
-      "reward": 0.7544643133878708,
-      "reward_std": 0.17125373054295778,
-      "rewards/accuracy_reward": 0.7544643133878708,
+      "loss": 0.0188,
+      "reward": 0.7366071864962578,
+      "reward_std": 0.19365339912474155,
+      "rewards/accuracy_reward": 0.7366071864962578,
       "rewards/format_reward": 0.0,
       "step": 49
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 601.2723503112793,
+      "completion_length": 598.0636329650879,
       "epoch": 0.8528784648187633,
-      "grad_norm": 0.1761648952960968,
-      "kl": 0.00411224365234375,
+      "grad_norm": 0.12470484524965286,
+      "kl": 0.0068721771240234375,
       "learning_rate": 1.718159615201853e-07,
-      "loss": 0.0408,
-      "reward": 0.7388393208384514,
-      "reward_std": 0.1724097654223442,
-      "rewards/accuracy_reward": 0.7388393208384514,
+      "loss": 0.0316,
+      "reward": 0.7433036044239998,
+      "reward_std": 0.18148937169462442,
+      "rewards/accuracy_reward": 0.7433036044239998,
       "rewards/format_reward": 0.0,
       "step": 50
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 568.6573905944824,
+      "completion_length": 563.3772583007812,
       "epoch": 0.8699360341151386,
-      "grad_norm": 0.2946619391441345,
-      "kl": 0.004871368408203125,
+      "grad_norm": 0.1378241628408432,
+      "kl": 0.0040340423583984375,
       "learning_rate": 1.321502310118649e-07,
-      "loss": 0.0445,
-      "reward": 0.805803619325161,
-      "reward_std": 0.16564481239765882,
-      "rewards/accuracy_reward": 0.805803619325161,
+      "loss": 0.0543,
+      "reward": 0.801339328289032,
+      "reward_std": 0.2019382379949093,
+      "rewards/accuracy_reward": 0.801339328289032,
       "rewards/format_reward": 0.0,
       "step": 51
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 621.2734756469727,
+      "completion_length": 614.0346298217773,
       "epoch": 0.8869936034115139,
-      "grad_norm": 0.11113651096820831,
-      "kl": 0.003757476806640625,
+      "grad_norm": 0.1643468588590622,
+      "kl": 0.004608154296875,
       "learning_rate": 9.747563597187792e-08,
-      "loss": 0.029,
-      "reward": 0.7589286044239998,
-      "reward_std": 0.20852677430957556,
-      "rewards/accuracy_reward": 0.7589286044239998,
+      "loss": 0.0382,
+      "reward": 0.7544643133878708,
+      "reward_std": 0.22550495527684689,
+      "rewards/accuracy_reward": 0.7544643133878708,
       "rewards/format_reward": 0.0,
       "step": 52
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 591.5591697692871,
+      "completion_length": 577.741081237793,
       "epoch": 0.9040511727078892,
-      "grad_norm": 0.10602231323719025,
-      "kl": 0.0047321319580078125,
+      "grad_norm": 0.12994147837162018,
+      "kl": 0.0049915313720703125,
       "learning_rate": 6.791870023718161e-08,
-      "loss": 0.0328,
-      "reward": 0.718750037252903,
-      "reward_std": 0.16462033987045288,
-      "rewards/accuracy_reward": 0.718750037252903,
+      "loss": 0.032,
+      "reward": 0.7176339626312256,
+      "reward_std": 0.19572455808520317,
+      "rewards/accuracy_reward": 0.7176339626312256,
       "rewards/format_reward": 0.0,
       "step": 53
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 569.8493576049805,
+      "completion_length": 555.170783996582,
       "epoch": 0.9211087420042644,
-      "grad_norm": 0.17479702830314636,
-      "kl": 0.0049591064453125,
+      "grad_norm": 0.14271694421768188,
+      "kl": 0.005153656005859375,
       "learning_rate": 4.358727386092198e-08,
-      "loss": 0.025,
-      "reward": 0.7957589700818062,
-      "reward_std": 0.19463301822543144,
-      "rewards/accuracy_reward": 0.7957589700818062,
+      "loss": 0.0232,
+      "reward": 0.8046875447034836,
+      "reward_std": 0.2259807214140892,
+      "rewards/accuracy_reward": 0.8046875447034836,
       "rewards/format_reward": 0.0,
       "step": 54
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 610.9352951049805,
+      "completion_length": 601.6172180175781,
       "epoch": 0.9381663113006397,
-      "grad_norm": 0.12260417640209198,
-      "kl": 0.0051116943359375,
+      "grad_norm": 0.15514791011810303,
+      "kl": 0.0044956207275390625,
       "learning_rate": 2.4570139579284723e-08,
-      "loss": 0.033,
-      "reward": 0.7979911118745804,
-      "reward_std": 0.1830651182681322,
-      "rewards/accuracy_reward": 0.7979911118745804,
+      "loss": 0.0323,
+      "reward": 0.8024554029107094,
+      "reward_std": 0.1908433297649026,
+      "rewards/accuracy_reward": 0.8024554029107094,
       "rewards/format_reward": 0.0,
       "step": 55
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 577.7366218566895,
+      "completion_length": 574.4933204650879,
       "epoch": 0.9552238805970149,
-      "grad_norm": 0.28994134068489075,
-      "kl": 0.0058689117431640625,
+      "grad_norm": 0.18661239743232727,
+      "kl": 0.006015777587890625,
       "learning_rate": 1.093668885291904e-08,
-      "loss": 0.0189,
-      "reward": 0.7857143208384514,
-      "reward_std": 0.1605983767658472,
-      "rewards/accuracy_reward": 0.7857143208384514,
+      "loss": 0.0185,
+      "reward": 0.7656250298023224,
+      "reward_std": 0.1824802029877901,
+      "rewards/accuracy_reward": 0.7656250298023224,
       "rewards/format_reward": 0.0,
       "step": 56
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 588.8772659301758,
+      "completion_length": 576.9375305175781,
       "epoch": 0.9722814498933902,
-      "grad_norm": 0.16521920263767242,
-      "kl": 0.00461578369140625,
+      "grad_norm": 0.15061481297016144,
+      "kl": 0.004833221435546875,
       "learning_rate": 2.736668665023756e-09,
-      "loss": 0.0413,
-      "reward": 0.7723214626312256,
-      "reward_std": 0.19243028853088617,
-      "rewards/accuracy_reward": 0.7723214626312256,
+      "loss": 0.0344,
+      "reward": 0.7745536118745804,
+      "reward_std": 0.2012776229530573,
+      "rewards/accuracy_reward": 0.7745536118745804,
       "rewards/format_reward": 0.0,
       "step": 57
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 586.1794052124023,
+      "completion_length": 588.2766265869141,
       "epoch": 0.9893390191897654,
-      "grad_norm": 0.31878846883773804,
-      "kl": 0.004428863525390625,
+      "grad_norm": 0.12136111408472061,
+      "kl": 0.0038356781005859375,
       "learning_rate": 0.0,
-      "loss": 0.0198,
-      "reward": 0.764508955180645,
-      "reward_std": 0.15518809761852026,
-      "rewards/accuracy_reward": 0.764508955180645,
+      "loss": 0.0255,
+      "reward": 0.7555803880095482,
+      "reward_std": 0.18643509317189455,
+      "rewards/accuracy_reward": 0.7555803880095482,
       "rewards/format_reward": 0.0,
       "step": 58
     },
+    {
+      "epoch": 0.9893390191897654,
+      "eval_clip_ratio": 0.0,
+      "eval_completion_length": 575.1875478726225,
+      "eval_kl": 0.011225008736022364,
+      "eval_loss": 0.02523801103234291,
+      "eval_reward": 0.673665024316349,
+      "eval_reward_std": 0.2331223714465912,
+      "eval_rewards/accuracy_reward": 0.6736364986378545,
+      "eval_rewards/format_reward": 2.8525788586931868e-05,
+      "eval_runtime": 4208.1068,
+      "eval_samples_per_second": 1.188,
+      "eval_steps_per_second": 0.011,
+      "step": 58
+    },
     {
       "epoch": 0.9893390191897654,
       "step": 58,
       "total_flos": 0.0,
-      "train_loss": 0.04100553294370786,
-      "train_runtime": 8329.5214,
-      "train_samples_per_second": 0.9,
-      "train_steps_per_second": 0.007
+      "train_loss": 0.04285794010000496,
+      "train_runtime": 11336.4845,
+      "train_samples_per_second": 0.662,
+      "train_steps_per_second": 0.005
     }
   ],
   "logging_steps": 1,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e9e03ec4cb080ac7b2fea9657a273e1b089c043de80aabdc3468383c37fb320
-size 7992
+oid sha256:c9d34ada9554116268a24d22a2e02b0b01443418c7b20a28d26ba8a9daedcdf9
+size 8056