Model save

README.md

@@ -27,7 +27,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/ligeng-zhu/openr1/runs/vdr82ewr) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/ligeng-zhu/openr1/runs/cy51opva) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0011118109807661124,
-    "train_runtime": 12310.5993,
+    "train_loss": -0.00014772719968559928,
+    "train_runtime": 12261.9552,
     "train_samples": 7500,
-    "train_samples_per_second": 0.609,
+    "train_samples_per_second": 0.612,
     "train_steps_per_second": 0.005
 }

model-00001-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cbf1ead14d78145d667212b8ac3487eb9fa5ff73e4cb6a8358492679d825929a
+oid sha256:3fc6f55571bcb95d956c7cfd35906a43b7b4fd1867862496319e593bc781eb20
 size 4877660776

model-00002-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1bdbbe9882c2c5a4f7d2fc7e6f2d285786c7cb3a31c4176c4e64b38432afa4b8
+oid sha256:52e0ae9a5a539b827261fdcb81a2b56701a1707f6960c3c3e145bae58042fc7c
 size 4932751008

model-00003-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1c94aaa97e6c96bf27f564944be524064e26fd0489128f5ec7beef219eba1ef4
+oid sha256:9e494758064a66d38bdfc7c5c6f3113a52a908d21dec04a41cf7abbe30735b23
 size 4330865200

model-00004-of-00004.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:296842bdd672617796c8ed82d8055cf9eb9448c72044d2a7e49b6617863ec8f4
+oid sha256:bfb2e4c79185c8542c9c7453a93b773a3a530bcfd583a45661655853795ecbc6
 size 1089994880

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.0011118109807661124,
-    "train_runtime": 12310.5993,
+    "train_loss": -0.00014772719968559928,
+    "train_runtime": 12261.9552,
     "train_samples": 7500,
-    "train_samples_per_second": 0.609,
+    "train_samples_per_second": 0.612,
     "train_steps_per_second": 0.005
 }

trainer_state.json

@@ -9,161 +9,161 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 612.6890861511231,
+      "completion_length": 614.0942222595215,
       "epoch": 0.08528784648187633,
-      "grad_norm": 499.6991882324219,
-      "kl": 0.08026816844940185,
+      "grad_norm": 39.48369216918945,
+      "kl": 0.002408742904663086,
       "learning_rate": 2.5e-06,
-      "loss": 0.0032,
-      "reward": 0.634151815623045,
-      "reward_std": 0.3310549106448889,
-      "rewards/accuracy_reward": 0.6339286021888256,
-      "rewards/format_reward": 0.00022321429569274187,
+      "loss": 0.0001,
+      "reward": 0.6303571693599224,
+      "reward_std": 0.3278109859675169,
+      "rewards/accuracy_reward": 0.6299107424914837,
+      "rewards/format_reward": 0.00044642859138548373,
       "step": 5
     },
     {
-      "completion_length": 622.5966781616211,
+      "completion_length": 618.1955627441406,
       "epoch": 0.17057569296375266,
-      "grad_norm": 1.3974753618240356,
-      "kl": 0.03140735626220703,
+      "grad_norm": 0.26418083906173706,
+      "kl": 0.0033366203308105467,
       "learning_rate": 2.956412726139078e-06,
-      "loss": 0.0013,
-      "reward": 0.6975446715950966,
-      "reward_std": 0.29021961949765684,
-      "rewards/accuracy_reward": 0.6968750283122063,
-      "rewards/format_reward": 0.0006696428870782256,
+      "loss": 0.0001,
+      "reward": 0.6823660999536514,
+      "reward_std": 0.2967432256788015,
+      "rewards/accuracy_reward": 0.6821428865194321,
+      "rewards/format_reward": 0.00022321429569274187,
       "step": 10
     },
     {
-      "completion_length": 620.2056060791016,
+      "completion_length": 615.4616325378418,
       "epoch": 0.255863539445629,
-      "grad_norm": 6.291464328765869,
-      "kl": 0.005489921569824219,
+      "grad_norm": 0.29952046275138855,
+      "kl": 0.004865837097167969,
       "learning_rate": 2.7836719084521715e-06,
       "loss": 0.0002,
-      "reward": 0.7453125342726707,
-      "reward_std": 0.23736504185944796,
-      "rewards/accuracy_reward": 0.7453125342726707,
+      "reward": 0.7544643208384514,
+      "reward_std": 0.22131893783807755,
+      "rewards/accuracy_reward": 0.7544643208384514,
       "rewards/format_reward": 0.0,
       "step": 15
     },
     {
-      "completion_length": 604.9377510070801,
+      "completion_length": 594.9377471923829,
       "epoch": 0.3411513859275053,
-      "grad_norm": 0.1435093730688095,
-      "kl": 0.0034137725830078124,
+      "grad_norm": 0.26904380321502686,
+      "kl": 0.0036653518676757813,
       "learning_rate": 2.4946839873611927e-06,
       "loss": 0.0001,
-      "reward": 0.7725446790456771,
-      "reward_std": 0.204142040386796,
-      "rewards/accuracy_reward": 0.7725446790456771,
+      "reward": 0.7558036029338837,
+      "reward_std": 0.20889290906488894,
+      "rewards/accuracy_reward": 0.7558036029338837,
       "rewards/format_reward": 0.0,
       "step": 20
     },
     {
-      "completion_length": 610.2647560119628,
+      "completion_length": 605.4913223266601,
       "epoch": 0.42643923240938164,
-      "grad_norm": 6.51456880569458,
-      "kl": 0.008059120178222657,
+      "grad_norm": 0.12021861970424652,
+      "kl": 0.003992271423339844,
       "learning_rate": 2.1156192081791355e-06,
-      "loss": 0.0003,
-      "reward": 0.7638393253087997,
-      "reward_std": 0.1955505058169365,
-      "rewards/accuracy_reward": 0.7638393253087997,
+      "loss": 0.0002,
+      "reward": 0.7569196805357933,
+      "reward_std": 0.19887337032705546,
+      "rewards/accuracy_reward": 0.7569196805357933,
       "rewards/format_reward": 0.0,
       "step": 25
     },
     {
-      "completion_length": 610.1558280944824,
+      "completion_length": 609.8823944091797,
       "epoch": 0.511727078891258,
-      "grad_norm": 0.07075676321983337,
-      "kl": 0.0029842376708984373,
+      "grad_norm": 0.09571157395839691,
+      "kl": 0.0034656524658203125,
       "learning_rate": 1.6808050203829845e-06,
       "loss": 0.0001,
-      "reward": 0.7504464626312256,
-      "reward_std": 0.18517553191632033,
-      "rewards/accuracy_reward": 0.7504464626312256,
+      "reward": 0.7562500357627868,
+      "reward_std": 0.1893269034102559,
+      "rewards/accuracy_reward": 0.7562500357627868,
       "rewards/format_reward": 0.0,
       "step": 30
     },
     {
-      "completion_length": 599.5821647644043,
+      "completion_length": 594.9317237854004,
       "epoch": 0.5970149253731343,
-      "grad_norm": 0.11963143199682236,
-      "kl": 0.002887916564941406,
+      "grad_norm": 0.17406129837036133,
+      "kl": 0.01944389343261719,
       "learning_rate": 1.2296174432791415e-06,
-      "loss": 0.0001,
-      "reward": 0.7441964641213417,
-      "reward_std": 0.1850216191262007,
-      "rewards/accuracy_reward": 0.7441964641213417,
+      "loss": 0.0008,
+      "reward": 0.7457589656114578,
+      "reward_std": 0.1772445771843195,
+      "rewards/accuracy_reward": 0.7457589656114578,
       "rewards/format_reward": 0.0,
       "step": 35
     },
     {
-      "completion_length": 585.4386421203614,
+      "completion_length": 587.7542655944824,
       "epoch": 0.6823027718550106,
-      "grad_norm": 0.12116171419620514,
-      "kl": 0.003470611572265625,
+      "grad_norm": 0.3893604278564453,
+      "kl": 0.0033367156982421877,
       "learning_rate": 8.029152419343472e-07,
       "loss": 0.0001,
-      "reward": 0.7640625357627868,
-      "reward_std": 0.17753117084503173,
-      "rewards/accuracy_reward": 0.7640625357627868,
+      "reward": 0.7665178909897804,
+      "reward_std": 0.17746288534253835,
+      "rewards/accuracy_reward": 0.7665178909897804,
       "rewards/format_reward": 0.0,
       "step": 40
     },
     {
-      "completion_length": 604.5248001098632,
+      "completion_length": 608.1580612182618,
       "epoch": 0.767590618336887,
-      "grad_norm": 0.18742159008979797,
-      "kl": 0.0029314041137695314,
+      "grad_norm": 0.13640980422496796,
+      "kl": 0.0030500411987304686,
       "learning_rate": 4.3933982822017883e-07,
       "loss": 0.0001,
-      "reward": 0.752678605914116,
-      "reward_std": 0.18469745945185423,
-      "rewards/accuracy_reward": 0.752678605914116,
+      "reward": 0.7549107506871223,
+      "reward_std": 0.18457430368289351,
+      "rewards/accuracy_reward": 0.7549107506871223,
       "rewards/format_reward": 0.0,
       "step": 45
     },
     {
-      "completion_length": 605.77993850708,
+      "completion_length": 602.2951156616211,
       "epoch": 0.8528784648187633,
-      "grad_norm": 0.10905129462480545,
-      "kl": 0.002948570251464844,
+      "grad_norm": 0.217549666762352,
+      "kl": 0.003106689453125,
       "learning_rate": 1.718159615201853e-07,
       "loss": 0.0001,
-      "reward": 0.7511161014437675,
-      "reward_std": 0.18544870764017105,
-      "rewards/accuracy_reward": 0.7511161014437675,
+      "reward": 0.7488839671015739,
+      "reward_std": 0.18157500196248294,
+      "rewards/accuracy_reward": 0.7488839671015739,
       "rewards/format_reward": 0.0,
       "step": 50
     },
     {
-      "completion_length": 598.5527030944825,
+      "completion_length": 599.9350708007812,
       "epoch": 0.9381663113006397,
-      "grad_norm": 0.11327774077653885,
-      "kl": 0.012871551513671874,
+      "grad_norm": 0.23885449767112732,
+      "kl": 0.0033361434936523436,
       "learning_rate": 2.4570139579284723e-08,
-      "loss": 0.0005,
-      "reward": 0.7783482506871223,
-      "reward_std": 0.19646856598556042,
-      "rewards/accuracy_reward": 0.7783482506871223,
+      "loss": 0.0001,
+      "reward": 0.776116107404232,
+      "reward_std": 0.19774878825992345,
+      "rewards/accuracy_reward": 0.776116107404232,
       "rewards/format_reward": 0.0,
       "step": 55
     },
     {
-      "completion_length": 588.8050842285156,
+      "completion_length": 583.1644630432129,
       "epoch": 0.9893390191897654,
-      "kl": 0.0034154256184895835,
-      "reward": 0.7719494377573332,
-      "reward_std": 0.1814334474814435,
-      "rewards/accuracy_reward": 0.7719494377573332,
+      "kl": 0.0035022099812825522,
+      "reward": 0.7760417039195696,
+      "reward_std": 0.16163485000530878,
+      "rewards/accuracy_reward": 0.7760417039195696,
       "rewards/format_reward": 0.0,
       "step": 58,
       "total_flos": 0.0,
-      "train_loss": 0.0011118109807661124,
-      "train_runtime": 12310.5993,
-      "train_samples_per_second": 0.609,
+      "train_loss": -0.00014772719968559928,
+      "train_runtime": 12261.9552,
+      "train_samples_per_second": 0.612,
       "train_steps_per_second": 0.005
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7a2b3dab96973f117202445b041ef4453ffe0e5c64232ffa41708ca0867bd02e
+oid sha256:ad87e5a39e36d78f4464aed52557072e88434fa4b15d87a8e6183f26c1addbbc
 size 7544