diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,2519 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9991024490319271, + "eval_steps": 400, + "global_step": 974, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "count/fg_chosen": 14.079999923706055, + "count/fg_rejected": 3.7826087474823, + "epoch": 0.01025772534940377, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.78272247314453, + "fg_logps/policy_chosen": -15.784832000732422, + "fg_logps/policy_rejected": -17.75761604309082, + "fg_logps/reference_KL": -21.78750991821289, + "fg_logps/reference_chosen": -15.790050506591797, + "fg_logps/reference_rejected": -17.756908416748047, + "fg_loss": 0.808033287525177, + "fg_rewards/chosen_sum": 0.006448890082538128, + "fg_rewards/rejected_sum": -2.7408803362050094e-05, + "grad_norm": 18.746104238140656, + "kl": 0.03848903253674507, + "learning_rate": 2.1220544726955587e-07, + "logps/chosen": -305.5322529560811, + "logps/rejected": -327.7266306322674, + "loss": 0.6362, + "rewards/chosen": -0.001747507501292873, + "rewards/margins": -0.002081668250248688, + "rewards/rejected": 0.00033416074895581535, + "step": 10 + }, + { + "count/fg_chosen": 13.947368621826172, + "count/fg_rejected": 3.5714285373687744, + "epoch": 0.02051545069880754, + "fg_kl": NaN, + "fg_logps/policy_KL": -20.42728614807129, + "fg_logps/policy_chosen": -15.204089164733887, + "fg_logps/policy_rejected": -20.7266902923584, + "fg_logps/reference_KL": -20.41632652282715, + "fg_logps/reference_chosen": -15.188560485839844, + "fg_logps/reference_rejected": -20.721162796020508, + "fg_loss": 0.7661333084106445, + "fg_rewards/chosen_sum": -0.025545625016093254, + "fg_rewards/rejected_sum": -0.00522112101316452, + "grad_norm": 19.071229786810708, + "kl": 0.006695461459457874, + "learning_rate": 3.0896765977473373e-07, + "logps/chosen": -313.60243626644734, + "logps/rejected": -306.7811569940476, + "loss": 0.5877, + "rewards/chosen": -0.012838767547356454, + "rewards/margins": 0.004141214908215037, + "rewards/rejected": -0.01697998245557149, + "step": 20 + }, + { + "count/fg_chosen": 15.894737243652344, + "count/fg_rejected": 3.5, + "epoch": 0.03077317604821131, + "fg_kl": NaN, + "fg_logps/policy_KL": -19.23096466064453, + "fg_logps/policy_chosen": -14.642033576965332, + "fg_logps/policy_rejected": -16.705078125, + "fg_logps/reference_KL": -19.1680850982666, + "fg_logps/reference_chosen": -14.53127384185791, + "fg_logps/reference_rejected": -16.613784790039062, + "fg_loss": 0.7854437232017517, + "fg_rewards/chosen_sum": -0.16305610537528992, + "fg_rewards/rejected_sum": -0.03273481875658035, + "grad_norm": 27.51124230932652, + "kl": 0.0037935494910925627, + "learning_rate": 3.5530203356711474e-07, + "logps/chosen": -328.168653612013, + "logps/rejected": -307.9456419427711, + "loss": 0.5906, + "rewards/chosen": -0.04671859121941901, + "rewards/margins": 0.03360887939913921, + "rewards/rejected": -0.08032747061855822, + "step": 30 + }, + { + "count/fg_chosen": 16.538461685180664, + "count/fg_rejected": 3.4761905670166016, + "epoch": 0.04103090139761508, + "fg_kl": NaN, + "fg_logps/policy_KL": -19.668180465698242, + "fg_logps/policy_chosen": -13.914729118347168, + "fg_logps/policy_rejected": -17.876625061035156, + "fg_logps/reference_KL": -19.54073143005371, + "fg_logps/reference_chosen": -13.71956729888916, + "fg_logps/reference_rejected": -17.639841079711914, + "fg_loss": 0.7382856011390686, + "fg_rewards/chosen_sum": -0.32776451110839844, + "fg_rewards/rejected_sum": -0.08581020683050156, + "grad_norm": 18.904689541215447, + "kl": 0.0, + "learning_rate": 3.907900061589056e-07, + "logps/chosen": -324.57538132440476, + "logps/rejected": -341.6456363075658, + "loss": 0.6186, + "rewards/chosen": -0.12434386071704683, + "rewards/margins": 0.07653144666724336, + "rewards/rejected": -0.20087530738429019, + "step": 40 + }, + { + "count/fg_chosen": 13.052631378173828, + "count/fg_rejected": 3.5999999046325684, + "epoch": 0.051288626747018846, + "fg_kl": NaN, + "fg_logps/policy_KL": -22.696481704711914, + "fg_logps/policy_chosen": -14.908551216125488, + "fg_logps/policy_rejected": -17.680747985839844, + "fg_logps/reference_KL": -22.48174476623535, + "fg_logps/reference_chosen": -14.603896141052246, + "fg_logps/reference_rejected": -17.34904670715332, + "fg_loss": 0.7404539585113525, + "fg_rewards/chosen_sum": -0.35476553440093994, + "fg_rewards/rejected_sum": -0.14453592896461487, + "grad_norm": 14.07450204568209, + "kl": 0.014550399966537952, + "learning_rate": 4.175211072161436e-07, + "logps/chosen": -291.7388392857143, + "logps/rejected": -314.5031061746988, + "loss": 0.5781, + "rewards/chosen": -0.10712227264007965, + "rewards/margins": 0.12518125482426995, + "rewards/rejected": -0.2323035274643496, + "step": 50 + }, + { + "count/fg_chosen": 14.5, + "count/fg_rejected": 4.166666507720947, + "epoch": 0.06154635209642262, + "fg_kl": NaN, + "fg_logps/policy_KL": -20.49053955078125, + "fg_logps/policy_chosen": -15.30705738067627, + "fg_logps/policy_rejected": -18.280315399169922, + "fg_logps/reference_KL": -20.184072494506836, + "fg_logps/reference_chosen": -14.819132804870605, + "fg_logps/reference_rejected": -17.81497573852539, + "fg_loss": 0.7277103066444397, + "fg_rewards/chosen_sum": -0.6670733690261841, + "fg_rewards/rejected_sum": -0.22372370958328247, + "grad_norm": 17.53900194639282, + "kl": 0.0, + "learning_rate": 4.389727636522206e-07, + "logps/chosen": -306.5228568412162, + "logps/rejected": -308.6946538880814, + "loss": 0.5582, + "rewards/chosen": -0.19411424688390783, + "rewards/margins": 0.22253594857053438, + "rewards/rejected": -0.4166501954544422, + "step": 60 + }, + { + "count/fg_chosen": 15.692307472229004, + "count/fg_rejected": 4.458333492279053, + "epoch": 0.07180407744582638, + "fg_kl": NaN, + "fg_logps/policy_KL": -20.50032615661621, + "fg_logps/policy_chosen": -15.366405487060547, + "fg_logps/policy_rejected": -20.129003524780273, + "fg_logps/reference_KL": -20.127044677734375, + "fg_logps/reference_chosen": -14.826613426208496, + "fg_logps/reference_rejected": -19.553565979003906, + "fg_loss": 0.7948136925697327, + "fg_rewards/chosen_sum": -0.8405532836914062, + "fg_rewards/rejected_sum": -0.24530810117721558, + "grad_norm": 17.707988179125024, + "kl": 0.016248703002929688, + "learning_rate": 4.568903445353587e-07, + "logps/chosen": -266.67440580985914, + "logps/rejected": -392.16322858146066, + "loss": 0.5804, + "rewards/chosen": -0.3093531433965119, + "rewards/margins": 0.4345650290326051, + "rewards/rejected": -0.743918172429117, + "step": 70 + }, + { + "count/fg_chosen": 13.82608699798584, + "count/fg_rejected": 4.736842155456543, + "epoch": 0.08206180279523016, + "fg_kl": NaN, + "fg_logps/policy_KL": -20.310537338256836, + "fg_logps/policy_chosen": -15.233660697937012, + "fg_logps/policy_rejected": -19.051939010620117, + "fg_logps/reference_KL": -19.766571044921875, + "fg_logps/reference_chosen": -14.48674201965332, + "fg_logps/reference_rejected": -18.337350845336914, + "fg_loss": 0.840079665184021, + "fg_rewards/chosen_sum": -1.008780837059021, + "fg_rewards/rejected_sum": -0.38555219769477844, + "grad_norm": 16.529452607844135, + "kl": 0.0, + "learning_rate": 4.7227525172567226e-07, + "logps/chosen": -258.29019579475306, + "logps/rejected": -339.7275761471519, + "loss": 0.5753, + "rewards/chosen": -0.38390738875777636, + "rewards/margins": 0.5281775528648007, + "rewards/rejected": -0.9120849416225771, + "step": 80 + }, + { + "count/fg_chosen": 16.94444465637207, + "count/fg_rejected": 4.5625, + "epoch": 0.09231952814463393, + "fg_kl": NaN, + "fg_logps/policy_KL": -18.532676696777344, + "fg_logps/policy_chosen": -14.168781280517578, + "fg_logps/policy_rejected": -15.193059921264648, + "fg_logps/reference_KL": -17.916645050048828, + "fg_logps/reference_chosen": -13.280251502990723, + "fg_logps/reference_rejected": -14.359517097473145, + "fg_loss": 0.7089979648590088, + "fg_rewards/chosen_sum": -1.4005531072616577, + "fg_rewards/rejected_sum": -0.3471396267414093, + "grad_norm": 22.36999453602448, + "kl": 0.013136100955307484, + "learning_rate": 4.857556042627658e-07, + "logps/chosen": -354.51475432981925, + "logps/rejected": -378.41106939935065, + "loss": 0.55, + "rewards/chosen": -0.5630912321159639, + "rewards/margins": 0.6156807405402861, + "rewards/rejected": -1.17877197265625, + "step": 90 + }, + { + "count/fg_chosen": 17.559999465942383, + "count/fg_rejected": 3.4583332538604736, + "epoch": 0.10257725349403769, + "fg_kl": NaN, + "fg_logps/policy_KL": -19.625595092773438, + "fg_logps/policy_chosen": -14.884417533874512, + "fg_logps/policy_rejected": -19.52889060974121, + "fg_logps/reference_KL": -18.972139358520508, + "fg_logps/reference_chosen": -13.96281623840332, + "fg_logps/reference_rejected": -18.573102951049805, + "fg_loss": 0.7564048171043396, + "fg_rewards/chosen_sum": -1.531907081604004, + "fg_rewards/rejected_sum": -0.3621842861175537, + "grad_norm": 16.9566179157687, + "kl": 0.0, + "learning_rate": 4.977514249230057e-07, + "logps/chosen": -296.1236049107143, + "logps/rejected": -352.51732113486844, + "loss": 0.6056, + "rewards/chosen": -0.8061084747314453, + "rewards/margins": 0.3998990309865851, + "rewards/rejected": -1.2060075057180304, + "step": 100 + }, + { + "count/fg_chosen": 17.423076629638672, + "count/fg_rejected": 3.277777671813965, + "epoch": 0.11283497884344147, + "fg_kl": NaN, + "fg_logps/policy_KL": -19.713420867919922, + "fg_logps/policy_chosen": -14.724455833435059, + "fg_logps/policy_rejected": -17.52227020263672, + "fg_logps/reference_KL": -19.24376678466797, + "fg_logps/reference_chosen": -14.040182113647461, + "fg_logps/reference_rejected": -16.54549789428711, + "fg_loss": 0.6636641621589661, + "fg_rewards/chosen_sum": -0.9770669341087341, + "fg_rewards/rejected_sum": -0.3046890199184418, + "grad_norm": 15.8810865349395, + "kl": 0.013418341055512428, + "learning_rate": 4.960045662100457e-07, + "logps/chosen": -319.42288602941176, + "logps/rejected": -328.1127604166667, + "loss": 0.5794, + "rewards/chosen": -0.6423200719496783, + "rewards/margins": 0.5826459518133426, + "rewards/rejected": -1.2249660237630209, + "step": 110 + }, + { + "count/fg_chosen": 17.038461685180664, + "count/fg_rejected": 3.549999952316284, + "epoch": 0.12309270419284524, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.378173828125, + "fg_logps/policy_chosen": -15.218153953552246, + "fg_logps/policy_rejected": -17.869609832763672, + "fg_logps/reference_KL": -20.997371673583984, + "fg_logps/reference_chosen": -14.897927284240723, + "fg_logps/reference_rejected": -17.389179229736328, + "fg_loss": 0.7092593908309937, + "fg_rewards/chosen_sum": -0.43586668372154236, + "fg_rewards/rejected_sum": -0.22385962307453156, + "grad_norm": 13.44458367896507, + "kl": 0.11229915916919708, + "learning_rate": 4.90296803652968e-07, + "logps/chosen": -279.3057077891791, + "logps/rejected": -315.11082409274195, + "loss": 0.5637, + "rewards/chosen": -0.42403762020281893, + "rewards/margins": 0.6563830071860841, + "rewards/rejected": -1.080420627388903, + "step": 120 + }, + { + "count/fg_chosen": 18.882352828979492, + "count/fg_rejected": 4.5333333015441895, + "epoch": 0.13335042954224902, + "fg_kl": NaN, + "fg_logps/policy_KL": -18.839466094970703, + "fg_logps/policy_chosen": -14.21582317352295, + "fg_logps/policy_rejected": -15.989607810974121, + "fg_logps/reference_KL": -18.35956382751465, + "fg_logps/reference_chosen": -13.778548240661621, + "fg_logps/reference_rejected": -15.061809539794922, + "fg_loss": 0.752821147441864, + "fg_rewards/chosen_sum": -0.353499174118042, + "fg_rewards/rejected_sum": -0.5261321663856506, + "grad_norm": 18.943079661475036, + "kl": 0.0, + "learning_rate": 4.845890410958904e-07, + "logps/chosen": -399.24955100574715, + "logps/rejected": -355.0519852311644, + "loss": 0.5522, + "rewards/chosen": -0.5169386589664152, + "rewards/margins": 0.7560970959297749, + "rewards/rejected": -1.2730357548961901, + "step": 130 + }, + { + "count/fg_chosen": 14.600000381469727, + "count/fg_rejected": 3.2941176891326904, + "epoch": 0.14360815489165277, + "fg_kl": NaN, + "fg_logps/policy_KL": -22.31153106689453, + "fg_logps/policy_chosen": -15.935779571533203, + "fg_logps/policy_rejected": -19.569046020507812, + "fg_logps/reference_KL": -21.87625503540039, + "fg_logps/reference_chosen": -15.854820251464844, + "fg_logps/reference_rejected": -19.18140983581543, + "fg_loss": 0.6795336604118347, + "fg_rewards/chosen_sum": 0.07530271261930466, + "fg_rewards/rejected_sum": -0.0755414366722107, + "grad_norm": 14.329744807366827, + "kl": 0.010508442297577858, + "learning_rate": 4.788812785388127e-07, + "logps/chosen": -288.93102254746833, + "logps/rejected": -339.4910059799383, + "loss": 0.5424, + "rewards/chosen": -0.38591092749486994, + "rewards/margins": 0.9102081973062006, + "rewards/rejected": -1.2961191248010706, + "step": 140 + }, + { + "count/fg_chosen": 17.049999237060547, + "count/fg_rejected": 3.2352941036224365, + "epoch": 0.15386588024105655, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.11859703063965, + "fg_logps/policy_chosen": -15.353170394897461, + "fg_logps/policy_rejected": -19.041038513183594, + "fg_logps/reference_KL": -20.675270080566406, + "fg_logps/reference_chosen": -15.19269847869873, + "fg_logps/reference_rejected": -18.65860366821289, + "fg_loss": 0.6720137596130371, + "fg_rewards/chosen_sum": -0.040867485105991364, + "fg_rewards/rejected_sum": -0.07588446885347366, + "grad_norm": 15.943835226437113, + "kl": 0.0, + "learning_rate": 4.7317351598173515e-07, + "logps/chosen": -372.51135706018516, + "logps/rejected": -344.8247132120253, + "loss": 0.5326, + "rewards/chosen": -0.4752860363618827, + "rewards/margins": 0.8493531029342507, + "rewards/rejected": -1.3246391392961334, + "step": 150 + }, + { + "count/fg_chosen": 14.833333015441895, + "count/fg_rejected": 2.642857074737549, + "epoch": 0.16412360559046033, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.43903350830078, + "fg_logps/policy_chosen": -14.500401496887207, + "fg_logps/policy_rejected": -17.249717712402344, + "fg_logps/reference_KL": -21.081066131591797, + "fg_logps/reference_chosen": -14.807108879089355, + "fg_logps/reference_rejected": -17.323772430419922, + "fg_loss": 0.6379384994506836, + "fg_rewards/chosen_sum": 0.5964276194572449, + "fg_rewards/rejected_sum": -0.0015845863381400704, + "grad_norm": 13.675438870382441, + "kl": 0.0012632369762286544, + "learning_rate": 4.6746575342465747e-07, + "logps/chosen": -327.4292566636029, + "logps/rejected": -389.55515455163044, + "loss": 0.4826, + "rewards/chosen": -0.45451857061947093, + "rewards/margins": 0.7520349300121103, + "rewards/rejected": -1.2065535006315813, + "step": 160 + }, + { + "count/fg_chosen": 18.764705657958984, + "count/fg_rejected": 3.538461446762085, + "epoch": 0.17438133093986408, + "fg_kl": NaN, + "fg_logps/policy_KL": -19.866031646728516, + "fg_logps/policy_chosen": -14.610063552856445, + "fg_logps/policy_rejected": -15.764613151550293, + "fg_logps/reference_KL": -19.758529663085938, + "fg_logps/reference_chosen": -15.149678230285645, + "fg_logps/reference_rejected": -15.872199058532715, + "fg_loss": 0.7423752546310425, + "fg_rewards/chosen_sum": 1.0474270582199097, + "fg_rewards/rejected_sum": -0.00038578076055273414, + "grad_norm": 14.85184229504073, + "kl": 0.05892143398523331, + "learning_rate": 4.617579908675799e-07, + "logps/chosen": -289.5171331091772, + "logps/rejected": -364.5406298225309, + "loss": 0.5105, + "rewards/chosen": -0.3850521618806863, + "rewards/margins": 0.9919037888209918, + "rewards/rejected": -1.3769559507016782, + "step": 170 + }, + { + "count/fg_chosen": 14.399999618530273, + "count/fg_rejected": 4.5, + "epoch": 0.18463905628926786, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.385639190673828, + "fg_logps/policy_chosen": -15.719804763793945, + "fg_logps/policy_rejected": -21.617740631103516, + "fg_logps/reference_KL": -21.118305206298828, + "fg_logps/reference_chosen": -15.717147827148438, + "fg_logps/reference_rejected": -21.633119583129883, + "fg_loss": 0.875866711139679, + "fg_rewards/chosen_sum": 0.0969744473695755, + "fg_rewards/rejected_sum": -0.13854043185710907, + "grad_norm": 14.688764944309868, + "kl": 0.0, + "learning_rate": 4.5605022831050226e-07, + "logps/chosen": -251.9897216796875, + "logps/rejected": -383.8666259765625, + "loss": 0.5364, + "rewards/chosen": -0.6086512088775635, + "rewards/margins": 1.0637765407562256, + "rewards/rejected": -1.6724277496337892, + "step": 180 + }, + { + "count/fg_chosen": 19.217391967773438, + "count/fg_rejected": 4.523809432983398, + "epoch": 0.19489678163867163, + "fg_kl": NaN, + "fg_logps/policy_KL": -18.718454360961914, + "fg_logps/policy_chosen": -13.899271965026855, + "fg_logps/policy_rejected": -18.827178955078125, + "fg_logps/reference_KL": -18.324739456176758, + "fg_logps/reference_chosen": -13.679468154907227, + "fg_logps/reference_rejected": -18.46164321899414, + "fg_loss": 0.7202942967414856, + "fg_rewards/chosen_sum": -0.006031676661223173, + "fg_rewards/rejected_sum": -0.15737372636795044, + "grad_norm": 14.404560894432748, + "kl": 0.05560264736413956, + "learning_rate": 4.503424657534247e-07, + "logps/chosen": -338.58002804487177, + "logps/rejected": -338.0175543064024, + "loss": 0.5344, + "rewards/chosen": -0.542063737526918, + "rewards/margins": 1.0375677887688135, + "rewards/rejected": -1.5796315262957317, + "step": 190 + }, + { + "count/fg_chosen": 17.399999618530273, + "count/fg_rejected": 4.789473533630371, + "epoch": 0.20515450698807539, + "fg_kl": NaN, + "fg_logps/policy_KL": -19.66611099243164, + "fg_logps/policy_chosen": -14.653355598449707, + "fg_logps/policy_rejected": -17.5184383392334, + "fg_logps/reference_KL": -19.25302505493164, + "fg_logps/reference_chosen": -14.066553115844727, + "fg_logps/reference_rejected": -17.042163848876953, + "fg_loss": 0.9406682848930359, + "fg_rewards/chosen_sum": -0.7670488357543945, + "fg_rewards/rejected_sum": -0.16450081765651703, + "grad_norm": 13.472817295849191, + "kl": 0.0, + "learning_rate": 4.44634703196347e-07, + "logps/chosen": -328.9538845486111, + "logps/rejected": -382.8152622767857, + "loss": 0.5533, + "rewards/chosen": -0.6610004001193577, + "rewards/margins": 1.229798332093254, + "rewards/rejected": -1.8907987322126116, + "step": 200 + }, + { + "count/fg_chosen": 17.6875, + "count/fg_rejected": 5.384615421295166, + "epoch": 0.21541223233747916, + "fg_kl": NaN, + "fg_logps/policy_KL": -20.451927185058594, + "fg_logps/policy_chosen": -14.831332206726074, + "fg_logps/policy_rejected": -17.854537963867188, + "fg_logps/reference_KL": -20.010757446289062, + "fg_logps/reference_chosen": -14.1827392578125, + "fg_logps/reference_rejected": -16.983478546142578, + "fg_loss": 0.7702024579048157, + "fg_rewards/chosen_sum": -1.0590859651565552, + "fg_rewards/rejected_sum": -0.4240546226501465, + "grad_norm": 16.379366840236408, + "kl": 0.0, + "learning_rate": 4.3892694063926936e-07, + "logps/chosen": -336.93881138392857, + "logps/rejected": -410.259375, + "loss": 0.4634, + "rewards/chosen": -0.4613551548549107, + "rewards/margins": 1.4628642733134922, + "rewards/rejected": -1.9242194281684029, + "step": 210 + }, + { + "count/fg_chosen": 14.0, + "count/fg_rejected": 4.181818008422852, + "epoch": 0.22566995768688294, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.021493911743164, + "fg_logps/policy_chosen": -16.28066062927246, + "fg_logps/policy_rejected": -20.48377799987793, + "fg_logps/reference_KL": -20.17574691772461, + "fg_logps/reference_chosen": -15.166167259216309, + "fg_logps/reference_rejected": -19.03717803955078, + "fg_loss": 0.6626110672950745, + "fg_rewards/chosen_sum": -1.1887147426605225, + "fg_rewards/rejected_sum": -0.47799989581108093, + "grad_norm": 13.99674523050283, + "kl": 0.0, + "learning_rate": 4.332191780821918e-07, + "logps/chosen": -288.56021278782896, + "logps/rejected": -361.5675455729167, + "loss": 0.4491, + "rewards/chosen": -0.5620980011789423, + "rewards/margins": 1.510199412964938, + "rewards/rejected": -2.0722974141438804, + "step": 220 + }, + { + "count/fg_chosen": 14.4399995803833, + "count/fg_rejected": 4.888888835906982, + "epoch": 0.2359276830362867, + "fg_kl": NaN, + "fg_logps/policy_KL": -20.916282653808594, + "fg_logps/policy_chosen": -15.752463340759277, + "fg_logps/policy_rejected": -19.13568878173828, + "fg_logps/reference_KL": -20.100528717041016, + "fg_logps/reference_chosen": -14.519099235534668, + "fg_logps/reference_rejected": -17.75797462463379, + "fg_loss": 0.8086566925048828, + "fg_rewards/chosen_sum": -1.4387943744659424, + "fg_rewards/rejected_sum": -0.7463889122009277, + "grad_norm": 9.096647980350944, + "kl": 0.0, + "learning_rate": 4.2751141552511415e-07, + "logps/chosen": -325.9561071908602, + "logps/rejected": -342.84818097014926, + "loss": 0.5797, + "rewards/chosen": -0.93502930671938, + "rewards/margins": 1.3941938706068793, + "rewards/rejected": -2.3292231773262593, + "step": 230 + }, + { + "count/fg_chosen": 19.450000762939453, + "count/fg_rejected": 3.882352828979492, + "epoch": 0.24618540838569047, + "fg_kl": NaN, + "fg_logps/policy_KL": -19.968975067138672, + "fg_logps/policy_chosen": -15.759946823120117, + "fg_logps/policy_rejected": -18.90660858154297, + "fg_logps/reference_KL": -19.11962890625, + "fg_logps/reference_chosen": -14.694534301757812, + "fg_logps/reference_rejected": -17.608003616333008, + "fg_loss": 0.7771142721176147, + "fg_rewards/chosen_sum": -2.0326945781707764, + "fg_rewards/rejected_sum": -0.5817491412162781, + "grad_norm": 13.085165433618942, + "kl": 0.0, + "learning_rate": 4.218036529680365e-07, + "logps/chosen": -324.1948939732143, + "logps/rejected": -355.4172092013889, + "loss": 0.4984, + "rewards/chosen": -0.8741681780133929, + "rewards/margins": 0.9406820145864335, + "rewards/rejected": -1.8148501925998264, + "step": 240 + }, + { + "count/fg_chosen": 15.941176414489746, + "count/fg_rejected": 3.5625, + "epoch": 0.25644313373509425, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.16470718383789, + "fg_logps/policy_chosen": -15.544363975524902, + "fg_logps/policy_rejected": -22.452640533447266, + "fg_logps/reference_KL": -20.34503746032715, + "fg_logps/reference_chosen": -14.364212989807129, + "fg_logps/reference_rejected": -20.861942291259766, + "fg_loss": 0.8010571599006653, + "fg_rewards/chosen_sum": -1.5489312410354614, + "fg_rewards/rejected_sum": -0.5739867687225342, + "grad_norm": 20.269756145960844, + "kl": 0.0004318237188272178, + "learning_rate": 4.160958904109589e-07, + "logps/chosen": -284.01116415895063, + "logps/rejected": -337.00729331487344, + "loss": 0.4967, + "rewards/chosen": -0.7318777390468267, + "rewards/margins": 1.3799085359235055, + "rewards/rejected": -2.1117862749703322, + "step": 250 + }, + { + "count/fg_chosen": 14.647058486938477, + "count/fg_rejected": 3.8333332538604736, + "epoch": 0.26670085908449803, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.48432159423828, + "fg_logps/policy_chosen": -15.442632675170898, + "fg_logps/policy_rejected": -21.998336791992188, + "fg_logps/reference_KL": -20.76389503479004, + "fg_logps/reference_chosen": -14.410895347595215, + "fg_logps/reference_rejected": -20.8482723236084, + "fg_loss": 0.7149848937988281, + "fg_rewards/chosen_sum": -0.9253503680229187, + "fg_rewards/rejected_sum": -0.5536249279975891, + "grad_norm": 13.117998935505918, + "kl": 0.0, + "learning_rate": 4.1038812785388125e-07, + "logps/chosen": -288.13616943359375, + "logps/rejected": -351.7694905598958, + "loss": 0.4464, + "rewards/chosen": -0.422292560338974, + "rewards/margins": 1.4216915667057037, + "rewards/rejected": -1.8439841270446777, + "step": 260 + }, + { + "count/fg_chosen": 14.785714149475098, + "count/fg_rejected": 3.642857074737549, + "epoch": 0.27695858443390176, + "fg_kl": NaN, + "fg_logps/policy_KL": -22.614913940429688, + "fg_logps/policy_chosen": -16.907217025756836, + "fg_logps/policy_rejected": -19.078645706176758, + "fg_logps/reference_KL": -21.76515007019043, + "fg_logps/reference_chosen": -16.108179092407227, + "fg_logps/reference_rejected": -18.381593704223633, + "fg_loss": 0.819545567035675, + "fg_rewards/chosen_sum": -0.7028852105140686, + "fg_rewards/rejected_sum": -0.34045127034187317, + "grad_norm": 18.77208905467823, + "kl": 0.0, + "learning_rate": 4.046803652968037e-07, + "logps/chosen": -345.2209114583333, + "logps/rejected": -358.41994485294117, + "loss": 0.475, + "rewards/chosen": -0.8343807983398438, + "rewards/margins": 1.4960064517750458, + "rewards/rejected": -2.3303872501148897, + "step": 270 + }, + { + "count/fg_chosen": 17.962963104248047, + "count/fg_rejected": 5.884615421295166, + "epoch": 0.28721630978330553, + "fg_kl": NaN, + "fg_logps/policy_KL": -20.402196884155273, + "fg_logps/policy_chosen": -15.454611778259277, + "fg_logps/policy_rejected": -17.580909729003906, + "fg_logps/reference_KL": -19.303863525390625, + "fg_logps/reference_chosen": -14.586993217468262, + "fg_logps/reference_rejected": -16.358680725097656, + "fg_loss": 0.8306005597114563, + "fg_rewards/chosen_sum": -1.1611218452453613, + "fg_rewards/rejected_sum": -0.7079381942749023, + "grad_norm": 13.80952507861355, + "kl": 0.0, + "learning_rate": 3.98972602739726e-07, + "logps/chosen": -283.1960693359375, + "logps/rejected": -412.07763671875, + "loss": 0.5757, + "rewards/chosen": -0.4877366542816162, + "rewards/margins": 1.6564238071441653, + "rewards/rejected": -2.1441604614257814, + "step": 280 + }, + { + "count/fg_chosen": 16.549999237060547, + "count/fg_rejected": 5.277777671813965, + "epoch": 0.2974740351327093, + "fg_kl": NaN, + "fg_logps/policy_KL": -22.114633560180664, + "fg_logps/policy_chosen": -15.825154304504395, + "fg_logps/policy_rejected": -20.11035919189453, + "fg_logps/reference_KL": -21.259445190429688, + "fg_logps/reference_chosen": -15.046856880187988, + "fg_logps/reference_rejected": -19.045265197753906, + "fg_loss": 0.8168869614601135, + "fg_rewards/chosen_sum": -0.7166315913200378, + "fg_rewards/rejected_sum": -0.4433734118938446, + "grad_norm": 14.177325128931766, + "kl": 0.007422161288559437, + "learning_rate": 3.9326484018264836e-07, + "logps/chosen": -319.059375, + "logps/rejected": -395.9953125, + "loss": 0.5099, + "rewards/chosen": -0.6542438761393229, + "rewards/margins": 1.4790777288698682, + "rewards/rejected": -2.133321605009191, + "step": 290 + }, + { + "count/fg_chosen": 13.82608699798584, + "count/fg_rejected": 4.7727274894714355, + "epoch": 0.3077317604821131, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.241731643676758, + "fg_logps/policy_chosen": -16.656099319458008, + "fg_logps/policy_rejected": -18.52686882019043, + "fg_logps/reference_KL": -20.161357879638672, + "fg_logps/reference_chosen": -15.559574127197266, + "fg_logps/reference_rejected": -17.188322067260742, + "fg_loss": 0.8407912850379944, + "fg_rewards/chosen_sum": -1.1080502271652222, + "fg_rewards/rejected_sum": -0.7194547057151794, + "grad_norm": 11.479402820786039, + "kl": 0.0, + "learning_rate": 3.875570776255708e-07, + "logps/chosen": -260.6196986607143, + "logps/rejected": -403.26040097891564, + "loss": 0.547, + "rewards/chosen": -1.1976228193803267, + "rewards/margins": 1.5034547452728173, + "rewards/rejected": -2.701077564653144, + "step": 300 + }, + { + "count/fg_chosen": 17.0, + "count/fg_rejected": 5.5333333015441895, + "epoch": 0.31798948583151687, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.364051818847656, + "fg_logps/policy_chosen": -15.91810417175293, + "fg_logps/policy_rejected": -18.95937728881836, + "fg_logps/reference_KL": -20.408035278320312, + "fg_logps/reference_chosen": -14.995894432067871, + "fg_logps/reference_rejected": -17.87520408630371, + "fg_loss": 0.8350895047187805, + "fg_rewards/chosen_sum": -1.1522879600524902, + "fg_rewards/rejected_sum": -0.6541081070899963, + "grad_norm": 12.005685770721653, + "kl": 0.05910682678222656, + "learning_rate": 3.8184931506849315e-07, + "logps/chosen": -314.4315476190476, + "logps/rejected": -347.2052580180921, + "loss": 0.5311, + "rewards/chosen": -0.8637207576206752, + "rewards/margins": 1.5000286962752951, + "rewards/rejected": -2.3637494538959705, + "step": 310 + }, + { + "count/fg_chosen": 14.047618865966797, + "count/fg_rejected": 5.5, + "epoch": 0.32824721118092065, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.606046676635742, + "fg_logps/policy_chosen": -15.302529335021973, + "fg_logps/policy_rejected": -20.181062698364258, + "fg_logps/reference_KL": -20.86503791809082, + "fg_logps/reference_chosen": -14.479813575744629, + "fg_logps/reference_rejected": -19.489940643310547, + "fg_loss": 0.7766517996788025, + "fg_rewards/chosen_sum": -0.8992184400558472, + "fg_rewards/rejected_sum": -0.4495273232460022, + "grad_norm": 16.284183960507036, + "kl": 0.0, + "learning_rate": 3.761415525114155e-07, + "logps/chosen": -260.18896484375, + "logps/rejected": -417.2262073863636, + "loss": 0.4704, + "rewards/chosen": -0.6337332195705838, + "rewards/margins": 1.9360473035561916, + "rewards/rejected": -2.5697805231267754, + "step": 320 + }, + { + "count/fg_chosen": 16.22222137451172, + "count/fg_rejected": 3.3333332538604736, + "epoch": 0.3385049365303244, + "fg_kl": NaN, + "fg_logps/policy_KL": -20.270763397216797, + "fg_logps/policy_chosen": -16.191619873046875, + "fg_logps/policy_rejected": -19.492420196533203, + "fg_logps/reference_KL": -19.561973571777344, + "fg_logps/reference_chosen": -15.197806358337402, + "fg_logps/reference_rejected": -18.365306854248047, + "fg_loss": 0.7132466435432434, + "fg_rewards/chosen_sum": -1.5309462547302246, + "fg_rewards/rejected_sum": -0.2937175929546356, + "grad_norm": 7.818574791129848, + "kl": 0.0, + "learning_rate": 3.704337899543379e-07, + "logps/chosen": -285.40846946022725, + "logps/rejected": -327.64527925531917, + "loss": 0.4679, + "rewards/chosen": -1.0337238889752012, + "rewards/margins": 1.5292926372673188, + "rewards/rejected": -2.56301652624252, + "step": 330 + }, + { + "count/fg_chosen": 16.36842155456543, + "count/fg_rejected": 8.75, + "epoch": 0.34876266187972815, + "fg_kl": NaN, + "fg_logps/policy_KL": -18.425670623779297, + "fg_logps/policy_chosen": -13.722508430480957, + "fg_logps/policy_rejected": -16.77787971496582, + "fg_logps/reference_KL": -17.679426193237305, + "fg_logps/reference_chosen": -13.155657768249512, + "fg_logps/reference_rejected": -15.926578521728516, + "fg_loss": 0.7438153624534607, + "fg_rewards/chosen_sum": -0.5837015509605408, + "fg_rewards/rejected_sum": -0.5333263874053955, + "grad_norm": 9.802323506414373, + "kl": 0.0, + "learning_rate": 3.6472602739726025e-07, + "logps/chosen": -300.561328125, + "logps/rejected": -314.146240234375, + "loss": 0.5005, + "rewards/chosen": -0.9040058135986329, + "rewards/margins": 1.4882362365722657, + "rewards/rejected": -2.3922420501708985, + "step": 340 + }, + { + "count/fg_chosen": 16.14285659790039, + "count/fg_rejected": 5.315789699554443, + "epoch": 0.35902038722913193, + "fg_kl": NaN, + "fg_logps/policy_KL": -20.929489135742188, + "fg_logps/policy_chosen": -15.678800582885742, + "fg_logps/policy_rejected": -19.949668884277344, + "fg_logps/reference_KL": -19.97319984436035, + "fg_logps/reference_chosen": -14.735365867614746, + "fg_logps/reference_rejected": -18.699697494506836, + "fg_loss": 0.9030398726463318, + "fg_rewards/chosen_sum": -1.2398407459259033, + "fg_rewards/rejected_sum": -0.8212930560112, + "grad_norm": 8.643717223258967, + "kl": 0.0, + "learning_rate": 3.590182648401826e-07, + "logps/chosen": -351.5288783482143, + "logps/rejected": -359.00362356085526, + "loss": 0.5488, + "rewards/chosen": -1.1266803741455078, + "rewards/margins": 1.3310967495566919, + "rewards/rejected": -2.4577771237021997, + "step": 350 + }, + { + "count/fg_chosen": 14.214285850524902, + "count/fg_rejected": 2.6666667461395264, + "epoch": 0.3692781125785357, + "fg_kl": NaN, + "fg_logps/policy_KL": -23.768299102783203, + "fg_logps/policy_chosen": -16.969907760620117, + "fg_logps/policy_rejected": -20.13111114501953, + "fg_logps/reference_KL": -22.658279418945312, + "fg_logps/reference_chosen": -15.871198654174805, + "fg_logps/reference_rejected": -18.806407928466797, + "fg_loss": 0.7307645678520203, + "fg_rewards/chosen_sum": -0.9898194670677185, + "fg_rewards/rejected_sum": -0.20603026449680328, + "grad_norm": 8.142135660669938, + "kl": 0.0, + "learning_rate": 3.53310502283105e-07, + "logps/chosen": -282.40988869863014, + "logps/rejected": -349.8757632902299, + "loss": 0.464, + "rewards/chosen": -1.211091551062179, + "rewards/margins": 1.277019747597573, + "rewards/rejected": -2.488111298659752, + "step": 360 + }, + { + "count/fg_chosen": 15.15999984741211, + "count/fg_rejected": 3.4166667461395264, + "epoch": 0.3795358379279395, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.647724151611328, + "fg_logps/policy_chosen": -15.575891494750977, + "fg_logps/policy_rejected": -18.71537971496582, + "fg_logps/reference_KL": -20.737442016601562, + "fg_logps/reference_chosen": -14.95889663696289, + "fg_logps/reference_rejected": -18.047760009765625, + "fg_loss": 0.7305776476860046, + "fg_rewards/chosen_sum": -0.11250638961791992, + "fg_rewards/rejected_sum": -0.22363699972629547, + "grad_norm": 10.677432473272551, + "kl": 0.09735889732837677, + "learning_rate": 3.476027397260274e-07, + "logps/chosen": -302.0405040922619, + "logps/rejected": -364.09403268914474, + "loss": 0.5405, + "rewards/chosen": -0.9889777047293526, + "rewards/margins": 1.4359847907733199, + "rewards/rejected": -2.4249624955026725, + "step": 370 + }, + { + "count/fg_chosen": 16.35714340209961, + "count/fg_rejected": 4.461538314819336, + "epoch": 0.38979356327734327, + "fg_kl": NaN, + "fg_logps/policy_KL": -23.035646438598633, + "fg_logps/policy_chosen": -16.462678909301758, + "fg_logps/policy_rejected": -19.290132522583008, + "fg_logps/reference_KL": -22.22718620300293, + "fg_logps/reference_chosen": -16.05594825744629, + "fg_logps/reference_rejected": -18.451143264770508, + "fg_loss": 0.772994875907898, + "fg_rewards/chosen_sum": -0.47333768010139465, + "fg_rewards/rejected_sum": -0.43138498067855835, + "grad_norm": 12.589055564062804, + "kl": 0.0, + "learning_rate": 3.418949771689498e-07, + "logps/chosen": -303.3091415777439, + "logps/rejected": -422.3029346955128, + "loss": 0.4738, + "rewards/chosen": -1.0245074760623094, + "rewards/margins": 1.969195730317899, + "rewards/rejected": -2.9937032063802085, + "step": 380 + }, + { + "count/fg_chosen": 15.047618865966797, + "count/fg_rejected": 3.6666667461395264, + "epoch": 0.400051288626747, + "fg_kl": NaN, + "fg_logps/policy_KL": -22.204946517944336, + "fg_logps/policy_chosen": -16.388538360595703, + "fg_logps/policy_rejected": -19.46855926513672, + "fg_logps/reference_KL": -21.356998443603516, + "fg_logps/reference_chosen": -15.460061073303223, + "fg_logps/reference_rejected": -17.982017517089844, + "fg_loss": 0.7031415700912476, + "fg_rewards/chosen_sum": -1.0803974866867065, + "fg_rewards/rejected_sum": -0.587272047996521, + "grad_norm": 13.352594752925569, + "kl": 0.013134384527802467, + "learning_rate": 3.361872146118721e-07, + "logps/chosen": -344.073916153169, + "logps/rejected": -408.09998244382024, + "loss": 0.5212, + "rewards/chosen": -1.473697286256602, + "rewards/margins": 1.0633102491849358, + "rewards/rejected": -2.537007535441538, + "step": 390 + }, + { + "count/fg_chosen": 14.714285850524902, + "count/fg_rejected": 3.444444417953491, + "epoch": 0.41030901397615077, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.212703704833984, + "fg_logps/policy_chosen": -15.624214172363281, + "fg_logps/policy_rejected": -18.56574249267578, + "fg_logps/reference_KL": -20.04758071899414, + "fg_logps/reference_chosen": -14.2003173828125, + "fg_logps/reference_rejected": -17.362810134887695, + "fg_loss": 0.7282993197441101, + "fg_rewards/chosen_sum": -1.263887643814087, + "fg_rewards/rejected_sum": -0.6069123148918152, + "grad_norm": 10.430038786087746, + "kl": 0.0, + "learning_rate": 3.304794520547945e-07, + "logps/chosen": -331.27660778985506, + "logps/rejected": -399.446042239011, + "loss": 0.4495, + "rewards/chosen": -0.9196471615114074, + "rewards/margins": 1.7936450546952378, + "rewards/rejected": -2.713292216206645, + "step": 400 + }, + { + "epoch": 0.41030901397615077, + "eval_count/fg_chosen": 16.01369857788086, + "eval_count/fg_rejected": 4.0824174880981445, + "eval_fg_kl": NaN, + "eval_fg_logps/policy_KL": -21.182559967041016, + "eval_fg_logps/policy_chosen": -16.106983184814453, + "eval_fg_logps/policy_rejected": -20.267147064208984, + "eval_fg_logps/reference_KL": -20.207021713256836, + "eval_fg_logps/reference_chosen": -14.929487228393555, + "eval_fg_logps/reference_rejected": -18.78682518005371, + "eval_fg_loss": 0.744895875453949, + "eval_fg_rewards/chosen_sum": -1.3183764219284058, + "eval_fg_rewards/rejected_sum": -0.5732491612434387, + "eval_kl": 0.005370728671550751, + "eval_logps/chosen": -303.5075557511737, + "eval_logps/rejected": -365.12120890022675, + "eval_loss": 0.4977550208568573, + "eval_rewards/chosen": -1.0396734246625587, + "eval_rewards/margins": 1.678511342733983, + "eval_rewards/rejected": -2.718184767396542, + "eval_runtime": 806.8942, + "eval_samples_per_second": 2.148, + "eval_steps_per_second": 1.074, + "step": 400 + }, + { + "count/fg_chosen": 13.666666984558105, + "count/fg_rejected": 3.3181817531585693, + "epoch": 0.42056673932555455, + "fg_kl": NaN, + "fg_logps/policy_KL": -26.42474937438965, + "fg_logps/policy_chosen": -20.587556838989258, + "fg_logps/policy_rejected": -19.216453552246094, + "fg_logps/reference_KL": -25.272415161132812, + "fg_logps/reference_chosen": -18.695642471313477, + "fg_logps/reference_rejected": -16.948997497558594, + "fg_loss": 0.8068851232528687, + "fg_rewards/chosen_sum": -1.4713115692138672, + "fg_rewards/rejected_sum": -0.7158498167991638, + "grad_norm": 7.617410625793688, + "kl": 0.0, + "learning_rate": 3.247716894977169e-07, + "logps/chosen": -285.80265184859155, + "logps/rejected": -366.92332338483146, + "loss": 0.4728, + "rewards/chosen": -0.8456243058325539, + "rewards/margins": 1.8824731597683062, + "rewards/rejected": -2.72809746560086, + "step": 410 + }, + { + "count/fg_chosen": 15.4375, + "count/fg_rejected": 3.799999952316284, + "epoch": 0.43082446467495833, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.31452178955078, + "fg_logps/policy_chosen": -16.454280853271484, + "fg_logps/policy_rejected": -22.019365310668945, + "fg_logps/reference_KL": -20.437578201293945, + "fg_logps/reference_chosen": -15.224520683288574, + "fg_logps/reference_rejected": -20.551807403564453, + "fg_loss": 0.8755480647087097, + "fg_rewards/chosen_sum": -1.7564505338668823, + "fg_rewards/rejected_sum": -0.6421544551849365, + "grad_norm": 14.132450428785305, + "kl": 0.0, + "learning_rate": 3.1906392694063925e-07, + "logps/chosen": -319.19919463734567, + "logps/rejected": -400.0357001582278, + "loss": 0.494, + "rewards/chosen": -1.2088648478190105, + "rewards/margins": 1.8420809653237902, + "rewards/rejected": -3.0509458131428007, + "step": 420 + }, + { + "count/fg_chosen": 15.55555534362793, + "count/fg_rejected": 3.8235294818878174, + "epoch": 0.4410821900243621, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.4989013671875, + "fg_logps/policy_chosen": -16.78192138671875, + "fg_logps/policy_rejected": -19.125896453857422, + "fg_logps/reference_KL": -20.556718826293945, + "fg_logps/reference_chosen": -15.185633659362793, + "fg_logps/reference_rejected": -17.72509002685547, + "fg_loss": 0.8559948205947876, + "fg_rewards/chosen_sum": -1.8476042747497559, + "fg_rewards/rejected_sum": -0.710954487323761, + "grad_norm": 11.780117349913754, + "kl": 0.0, + "learning_rate": 3.133561643835616e-07, + "logps/chosen": -301.3424876143293, + "logps/rejected": -394.38636818910254, + "loss": 0.4855, + "rewards/chosen": -0.9122947134622713, + "rewards/margins": 2.028106999591114, + "rewards/rejected": -2.9404017130533853, + "step": 430 + }, + { + "count/fg_chosen": 16.0, + "count/fg_rejected": 4.75, + "epoch": 0.4513399153737659, + "fg_kl": NaN, + "fg_logps/policy_KL": -18.407926559448242, + "fg_logps/policy_chosen": -14.23660945892334, + "fg_logps/policy_rejected": -19.2972354888916, + "fg_logps/reference_KL": -17.730350494384766, + "fg_logps/reference_chosen": -13.197031021118164, + "fg_logps/reference_rejected": -17.424985885620117, + "fg_loss": 0.9003034234046936, + "fg_rewards/chosen_sum": -1.1896860599517822, + "fg_rewards/rejected_sum": -0.8951053023338318, + "grad_norm": 13.19151786370417, + "kl": 0.0, + "learning_rate": 3.07648401826484e-07, + "logps/chosen": -381.12164834104937, + "logps/rejected": -368.0486797863924, + "loss": 0.4868, + "rewards/chosen": -1.618426099235629, + "rewards/margins": 0.9258255479409927, + "rewards/rejected": -2.5442516471766217, + "step": 440 + }, + { + "count/fg_chosen": 14.588234901428223, + "count/fg_rejected": 4.25, + "epoch": 0.4615976407231696, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.48902130126953, + "fg_logps/policy_chosen": -16.183223724365234, + "fg_logps/policy_rejected": -20.021495819091797, + "fg_logps/reference_KL": -20.65131950378418, + "fg_logps/reference_chosen": -15.696285247802734, + "fg_logps/reference_rejected": -18.851573944091797, + "fg_loss": 0.8272340297698975, + "fg_rewards/chosen_sum": -0.4996866285800934, + "fg_rewards/rejected_sum": -0.5080854296684265, + "grad_norm": 11.534503591617172, + "kl": 0.038506411015987396, + "learning_rate": 3.019406392694064e-07, + "logps/chosen": -257.8545692845395, + "logps/rejected": -389.2383277529762, + "loss": 0.4618, + "rewards/chosen": -1.0474451968544407, + "rewards/margins": 1.9993883864323896, + "rewards/rejected": -3.0468335832868303, + "step": 450 + }, + { + "count/fg_chosen": 13.0, + "count/fg_rejected": 3.5999999046325684, + "epoch": 0.4718553660725734, + "fg_kl": NaN, + "fg_logps/policy_KL": -25.045570373535156, + "fg_logps/policy_chosen": -18.018939971923828, + "fg_logps/policy_rejected": -24.219295501708984, + "fg_logps/reference_KL": -23.617990493774414, + "fg_logps/reference_chosen": -16.746540069580078, + "fg_logps/reference_rejected": -22.393386840820312, + "fg_loss": 0.7469481825828552, + "fg_rewards/chosen_sum": -0.8979846239089966, + "fg_rewards/rejected_sum": -0.5428987145423889, + "grad_norm": 9.60552438901234, + "kl": 0.045855142176151276, + "learning_rate": 2.9623287671232877e-07, + "logps/chosen": -297.8484563253012, + "logps/rejected": -350.15962357954544, + "loss": 0.4823, + "rewards/chosen": -1.071040233933782, + "rewards/margins": 1.3700017867896352, + "rewards/rejected": -2.441042020723417, + "step": 460 + }, + { + "count/fg_chosen": 18.479999542236328, + "count/fg_rejected": 2.7727272510528564, + "epoch": 0.48211309142197717, + "fg_kl": NaN, + "fg_logps/policy_KL": -18.86458969116211, + "fg_logps/policy_chosen": -13.49354362487793, + "fg_logps/policy_rejected": -17.55750274658203, + "fg_logps/reference_KL": -18.0754337310791, + "fg_logps/reference_chosen": -13.031634330749512, + "fg_logps/reference_rejected": -17.194589614868164, + "fg_loss": 0.6403311491012573, + "fg_rewards/chosen_sum": -0.32538533210754395, + "fg_rewards/rejected_sum": -0.19457949697971344, + "grad_norm": 10.665362097968403, + "kl": 0.0, + "learning_rate": 2.905251141552511e-07, + "logps/chosen": -284.00316540948273, + "logps/rejected": -342.78692208904107, + "loss": 0.5004, + "rewards/chosen": -0.6770254990150189, + "rewards/margins": 2.199749119055401, + "rewards/rejected": -2.8767746180704195, + "step": 470 + }, + { + "count/fg_chosen": 15.523809432983398, + "count/fg_rejected": 4.5714287757873535, + "epoch": 0.49237081677138095, + "fg_kl": NaN, + "fg_logps/policy_KL": -20.194419860839844, + "fg_logps/policy_chosen": -15.528361320495605, + "fg_logps/policy_rejected": -17.91610336303711, + "fg_logps/reference_KL": -19.58913803100586, + "fg_logps/reference_chosen": -15.298972129821777, + "fg_logps/reference_rejected": -17.643596649169922, + "fg_loss": 0.7117694616317749, + "fg_rewards/chosen_sum": 0.15593282878398895, + "fg_rewards/rejected_sum": -0.3202955722808838, + "grad_norm": 8.89018776362184, + "kl": 0.0, + "learning_rate": 2.848173515981735e-07, + "logps/chosen": -366.11363389756946, + "logps/rejected": -336.78042879971593, + "loss": 0.5079, + "rewards/chosen": -1.2623120413886175, + "rewards/margins": 1.1003893746270075, + "rewards/rejected": -2.362701416015625, + "step": 480 + }, + { + "count/fg_chosen": 18.47058868408203, + "count/fg_rejected": 2.3333332538604736, + "epoch": 0.5026285421207847, + "fg_kl": NaN, + "fg_logps/policy_KL": -22.13791275024414, + "fg_logps/policy_chosen": -15.788673400878906, + "fg_logps/policy_rejected": -16.09772491455078, + "fg_logps/reference_KL": -21.38779640197754, + "fg_logps/reference_chosen": -15.412402153015137, + "fg_logps/reference_rejected": -15.27255916595459, + "fg_loss": 0.5969161987304688, + "fg_rewards/chosen_sum": -0.09672492742538452, + "fg_rewards/rejected_sum": -0.2052246332168579, + "grad_norm": 9.190876919128005, + "kl": 0.0, + "learning_rate": 2.791095890410959e-07, + "logps/chosen": -293.7143950591216, + "logps/rejected": -421.82026707848837, + "loss": 0.454, + "rewards/chosen": -0.9402487471296981, + "rewards/margins": 2.0355014884943934, + "rewards/rejected": -2.9757502356240915, + "step": 490 + }, + { + "count/fg_chosen": 19.263158798217773, + "count/fg_rejected": 3.1764705181121826, + "epoch": 0.5128862674701885, + "fg_kl": NaN, + "fg_logps/policy_KL": -18.44475746154785, + "fg_logps/policy_chosen": -13.578866958618164, + "fg_logps/policy_rejected": -16.77104949951172, + "fg_logps/reference_KL": -18.07316780090332, + "fg_logps/reference_chosen": -13.79211139678955, + "fg_logps/reference_rejected": -16.677942276000977, + "fg_loss": 0.8001216650009155, + "fg_rewards/chosen_sum": 0.6667798161506653, + "fg_rewards/rejected_sum": -0.12318305671215057, + "grad_norm": 11.958302083795623, + "kl": 0.06755809485912323, + "learning_rate": 2.734018264840183e-07, + "logps/chosen": -306.4125142911585, + "logps/rejected": -373.8045873397436, + "loss": 0.4904, + "rewards/chosen": -0.8678309742997332, + "rewards/margins": 1.8353534958525104, + "rewards/rejected": -2.7031844701522436, + "step": 500 + }, + { + "count/fg_chosen": 17.157894134521484, + "count/fg_rejected": 3.8888888359069824, + "epoch": 0.5231439928195922, + "fg_kl": NaN, + "fg_logps/policy_KL": -20.518945693969727, + "fg_logps/policy_chosen": -14.510261535644531, + "fg_logps/policy_rejected": -19.071426391601562, + "fg_logps/reference_KL": -19.78034782409668, + "fg_logps/reference_chosen": -15.087010383605957, + "fg_logps/reference_rejected": -18.8708553314209, + "fg_loss": 0.7365949749946594, + "fg_rewards/chosen_sum": 1.3324851989746094, + "fg_rewards/rejected_sum": -0.13402292132377625, + "grad_norm": 11.840579393816673, + "kl": 0.0, + "learning_rate": 2.676940639269406e-07, + "logps/chosen": -276.75473257211536, + "logps/rejected": -376.3443216463415, + "loss": 0.4727, + "rewards/chosen": -0.42943524091671675, + "rewards/margins": 2.161930108085284, + "rewards/rejected": -2.5913653490020008, + "step": 510 + }, + { + "count/fg_chosen": 14.416666984558105, + "count/fg_rejected": 4.473684310913086, + "epoch": 0.5334017181689961, + "fg_kl": NaN, + "fg_logps/policy_KL": -23.022239685058594, + "fg_logps/policy_chosen": -15.866630554199219, + "fg_logps/policy_rejected": -18.78508949279785, + "fg_logps/reference_KL": -21.714487075805664, + "fg_logps/reference_chosen": -15.187236785888672, + "fg_logps/reference_rejected": -18.08281898498535, + "fg_loss": 0.7238275408744812, + "fg_rewards/chosen_sum": -0.16382475197315216, + "fg_rewards/rejected_sum": -0.5171225070953369, + "grad_norm": 12.176540624619156, + "kl": 0.04069461673498154, + "learning_rate": 2.61986301369863e-07, + "logps/chosen": -308.16112012987014, + "logps/rejected": -321.8852362575301, + "loss": 0.5229, + "rewards/chosen": -0.8690564044110187, + "rewards/margins": 1.6282350686099714, + "rewards/rejected": -2.49729147302099, + "step": 520 + }, + { + "count/fg_chosen": 16.81818199157715, + "count/fg_rejected": 3.588235378265381, + "epoch": 0.5436594435183998, + "fg_kl": NaN, + "fg_logps/policy_KL": -20.363672256469727, + "fg_logps/policy_chosen": -14.961847305297852, + "fg_logps/policy_rejected": -21.47898292541504, + "fg_logps/reference_KL": -19.768022537231445, + "fg_logps/reference_chosen": -14.601174354553223, + "fg_logps/reference_rejected": -20.11042594909668, + "fg_loss": 0.6309139728546143, + "fg_rewards/chosen_sum": 0.3970043957233429, + "fg_rewards/rejected_sum": -0.6216399669647217, + "grad_norm": 9.438462165579672, + "kl": 0.017117690294981003, + "learning_rate": 2.562785388127854e-07, + "logps/chosen": -286.2251333841463, + "logps/rejected": -331.3464793669872, + "loss": 0.512, + "rewards/chosen": -1.0500634356242855, + "rewards/margins": 1.6451627935298612, + "rewards/rejected": -2.6952262291541467, + "step": 530 + }, + { + "count/fg_chosen": 15.850000381469727, + "count/fg_rejected": 2.6875, + "epoch": 0.5539171688678035, + "fg_kl": NaN, + "fg_logps/policy_KL": -19.712533950805664, + "fg_logps/policy_chosen": -14.07727336883545, + "fg_logps/policy_rejected": -18.43448829650879, + "fg_logps/reference_KL": -18.728797912597656, + "fg_logps/reference_chosen": -13.999521255493164, + "fg_logps/reference_rejected": -17.450742721557617, + "fg_loss": 0.6835896372795105, + "fg_rewards/chosen_sum": 0.26210981607437134, + "fg_rewards/rejected_sum": -0.32241931557655334, + "grad_norm": 12.291319609660407, + "kl": 0.0, + "learning_rate": 2.5057077625570777e-07, + "logps/chosen": -306.774658203125, + "logps/rejected": -356.60145399305554, + "loss": 0.5117, + "rewards/chosen": -0.8661181709983132, + "rewards/margins": 1.7041260883061573, + "rewards/rejected": -2.5702442593044705, + "step": 540 + }, + { + "count/fg_chosen": 18.714284896850586, + "count/fg_rejected": 4.722222328186035, + "epoch": 0.5641748942172073, + "fg_kl": NaN, + "fg_logps/policy_KL": -19.954633712768555, + "fg_logps/policy_chosen": -13.810935974121094, + "fg_logps/policy_rejected": -17.9754638671875, + "fg_logps/reference_KL": -19.242063522338867, + "fg_logps/reference_chosen": -13.879505157470703, + "fg_logps/reference_rejected": -17.572124481201172, + "fg_loss": 0.7275994420051575, + "fg_rewards/chosen_sum": 0.31742504239082336, + "fg_rewards/rejected_sum": -0.11265398561954498, + "grad_norm": 8.68692731946405, + "kl": 0.03491802141070366, + "learning_rate": 2.4486301369863014e-07, + "logps/chosen": -307.91368272569446, + "logps/rejected": -344.2369939630682, + "loss": 0.4965, + "rewards/chosen": -0.7141922314961752, + "rewards/margins": 1.454848968621456, + "rewards/rejected": -2.1690412001176314, + "step": 550 + }, + { + "count/fg_chosen": 17.190475463867188, + "count/fg_rejected": 3.5, + "epoch": 0.5744326195666111, + "fg_kl": NaN, + "fg_logps/policy_KL": -19.567123413085938, + "fg_logps/policy_chosen": -14.335834503173828, + "fg_logps/policy_rejected": -19.222152709960938, + "fg_logps/reference_KL": -19.11014747619629, + "fg_logps/reference_chosen": -14.28973388671875, + "fg_logps/reference_rejected": -18.501300811767578, + "fg_loss": 0.7595401406288147, + "fg_rewards/chosen_sum": 0.21668264269828796, + "fg_rewards/rejected_sum": -0.24018199741840363, + "grad_norm": 13.295487655176196, + "kl": 0.01426544226706028, + "learning_rate": 2.391552511415525e-07, + "logps/chosen": -340.173639871988, + "logps/rejected": -346.4476207386364, + "loss": 0.5126, + "rewards/chosen": -0.6746720463396555, + "rewards/margins": 1.3085595353111157, + "rewards/rejected": -1.983231581650771, + "step": 560 + }, + { + "count/fg_chosen": 15.61111068725586, + "count/fg_rejected": 3.230769157409668, + "epoch": 0.5846903449160149, + "fg_kl": NaN, + "fg_logps/policy_KL": -20.59079933166504, + "fg_logps/policy_chosen": -15.83668327331543, + "fg_logps/policy_rejected": -20.33367919921875, + "fg_logps/reference_KL": -19.622900009155273, + "fg_logps/reference_chosen": -14.985246658325195, + "fg_logps/reference_rejected": -19.151033401489258, + "fg_loss": 0.7142822742462158, + "fg_rewards/chosen_sum": -1.0015321969985962, + "fg_rewards/rejected_sum": -0.4118236005306244, + "grad_norm": 14.880021607580103, + "kl": 0.0, + "learning_rate": 2.3344748858447487e-07, + "logps/chosen": -321.2857349537037, + "logps/rejected": -375.5545638844937, + "loss": 0.4752, + "rewards/chosen": -1.0196603845666956, + "rewards/margins": 1.8235435825639412, + "rewards/rejected": -2.843203967130637, + "step": 570 + }, + { + "count/fg_chosen": 14.615385055541992, + "count/fg_rejected": 3.2727272510528564, + "epoch": 0.5949480702654186, + "fg_kl": NaN, + "fg_logps/policy_KL": -22.73748779296875, + "fg_logps/policy_chosen": -16.519458770751953, + "fg_logps/policy_rejected": -19.338417053222656, + "fg_logps/reference_KL": -21.60148048400879, + "fg_logps/reference_chosen": -15.273176193237305, + "fg_logps/reference_rejected": -18.030052185058594, + "fg_loss": 0.7020931243896484, + "fg_rewards/chosen_sum": -1.3088314533233643, + "fg_rewards/rejected_sum": -0.47078707814216614, + "grad_norm": 16.46811452693527, + "kl": 0.010451125912368298, + "learning_rate": 2.2773972602739724e-07, + "logps/chosen": -263.2815152391975, + "logps/rejected": -350.35057357594934, + "loss": 0.531, + "rewards/chosen": -1.0503113358109086, + "rewards/margins": 1.5224891194031933, + "rewards/rejected": -2.572800455214102, + "step": 580 + }, + { + "count/fg_chosen": 15.391304016113281, + "count/fg_rejected": 3.941176414489746, + "epoch": 0.6052057956148225, + "fg_kl": NaN, + "fg_logps/policy_KL": -22.143524169921875, + "fg_logps/policy_chosen": -16.308269500732422, + "fg_logps/policy_rejected": -21.572189331054688, + "fg_logps/reference_KL": -20.941722869873047, + "fg_logps/reference_chosen": -15.216353416442871, + "fg_logps/reference_rejected": -19.638399124145508, + "fg_loss": 0.7051105499267578, + "fg_rewards/chosen_sum": -1.149840235710144, + "fg_rewards/rejected_sum": -0.7558993101119995, + "grad_norm": 10.114785462372456, + "kl": 0.0, + "learning_rate": 2.2203196347031963e-07, + "logps/chosen": -397.5863940746753, + "logps/rejected": -363.87947100903614, + "loss": 0.4982, + "rewards/chosen": -1.2836351270799513, + "rewards/margins": 1.5669618408699733, + "rewards/rejected": -2.8505969679499246, + "step": 590 + }, + { + "count/fg_chosen": 13.0, + "count/fg_rejected": 3.461538553237915, + "epoch": 0.6154635209642262, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.54435920715332, + "fg_logps/policy_chosen": -16.62588119506836, + "fg_logps/policy_rejected": -24.728574752807617, + "fg_logps/reference_KL": -20.335765838623047, + "fg_logps/reference_chosen": -15.187963485717773, + "fg_logps/reference_rejected": -22.167306900024414, + "fg_loss": 0.6712305545806885, + "fg_rewards/chosen_sum": -1.496664047241211, + "fg_rewards/rejected_sum": -0.9014762043952942, + "grad_norm": 13.343339163568743, + "kl": 0.0, + "learning_rate": 2.16324200913242e-07, + "logps/chosen": -308.1182432432432, + "logps/rejected": -434.2431413517442, + "loss": 0.4397, + "rewards/chosen": -0.7798370155128272, + "rewards/margins": 2.0589501520584848, + "rewards/rejected": -2.838787167571312, + "step": 600 + }, + { + "count/fg_chosen": 14.222222328186035, + "count/fg_rejected": 4.470588207244873, + "epoch": 0.6257212463136299, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.16152572631836, + "fg_logps/policy_chosen": -16.751249313354492, + "fg_logps/policy_rejected": -19.572917938232422, + "fg_logps/reference_KL": -20.091609954833984, + "fg_logps/reference_chosen": -15.278923988342285, + "fg_logps/reference_rejected": -17.713302612304688, + "fg_loss": 0.7949765920639038, + "fg_rewards/chosen_sum": -1.593385100364685, + "fg_rewards/rejected_sum": -0.8538177609443665, + "grad_norm": 13.466395129856458, + "kl": 0.0, + "learning_rate": 2.106164383561644e-07, + "logps/chosen": -293.43453125, + "logps/rejected": -324.9224264705882, + "loss": 0.4617, + "rewards/chosen": -1.0275954182942708, + "rewards/margins": 1.7896552710439646, + "rewards/rejected": -2.8172506893382354, + "step": 610 + }, + { + "count/fg_chosen": 18.760000228881836, + "count/fg_rejected": 4.304347991943359, + "epoch": 0.6359789716630337, + "fg_kl": NaN, + "fg_logps/policy_KL": -18.953258514404297, + "fg_logps/policy_chosen": -14.654019355773926, + "fg_logps/policy_rejected": -18.075929641723633, + "fg_logps/reference_KL": -18.280025482177734, + "fg_logps/reference_chosen": -14.031706809997559, + "fg_logps/reference_rejected": -16.499387741088867, + "fg_loss": 0.753377377986908, + "fg_rewards/chosen_sum": -0.787641704082489, + "fg_rewards/rejected_sum": -0.6412522792816162, + "grad_norm": 11.259123987500764, + "kl": 0.0, + "learning_rate": 2.0490867579908674e-07, + "logps/chosen": -303.9673755787037, + "logps/rejected": -421.6932357594937, + "loss": 0.4916, + "rewards/chosen": -0.5601168502995997, + "rewards/margins": 2.57166944784864, + "rewards/rejected": -3.13178629814824, + "step": 620 + }, + { + "count/fg_chosen": 19.100000381469727, + "count/fg_rejected": 3.9375, + "epoch": 0.6462366970124375, + "fg_kl": NaN, + "fg_logps/policy_KL": -17.8834171295166, + "fg_logps/policy_chosen": -12.904516220092773, + "fg_logps/policy_rejected": -18.650463104248047, + "fg_logps/reference_KL": -17.336223602294922, + "fg_logps/reference_chosen": -12.578201293945312, + "fg_logps/reference_rejected": -18.051301956176758, + "fg_loss": 0.8024751543998718, + "fg_rewards/chosen_sum": -0.19778959453105927, + "fg_rewards/rejected_sum": -0.45491790771484375, + "grad_norm": 11.633599458809858, + "kl": 0.0, + "learning_rate": 1.9920091324200913e-07, + "logps/chosen": -319.94747740963857, + "logps/rejected": -482.38976258116884, + "loss": 0.4819, + "rewards/chosen": -0.8397377657603069, + "rewards/margins": 2.030176697827863, + "rewards/rejected": -2.8699144635881697, + "step": 630 + }, + { + "count/fg_chosen": 14.666666984558105, + "count/fg_rejected": 4.599999904632568, + "epoch": 0.6564944223618413, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.435789108276367, + "fg_logps/policy_chosen": -14.770197868347168, + "fg_logps/policy_rejected": -21.445520401000977, + "fg_logps/reference_KL": -20.57337760925293, + "fg_logps/reference_chosen": -14.659423828125, + "fg_logps/reference_rejected": -20.503273010253906, + "fg_loss": 0.7505216598510742, + "fg_rewards/chosen_sum": 0.48458048701286316, + "fg_rewards/rejected_sum": -0.5640232563018799, + "grad_norm": 10.675237142753284, + "kl": 0.0, + "learning_rate": 1.934931506849315e-07, + "logps/chosen": -296.4902083333333, + "logps/rejected": -395.98147977941176, + "loss": 0.4865, + "rewards/chosen": -0.7284186808268229, + "rewards/margins": 1.9219920498717067, + "rewards/rejected": -2.6504107306985296, + "step": 640 + }, + { + "count/fg_chosen": 13.600000381469727, + "count/fg_rejected": 3.642857074737549, + "epoch": 0.666752147711245, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.876583099365234, + "fg_logps/policy_chosen": -15.880990982055664, + "fg_logps/policy_rejected": -18.033313751220703, + "fg_logps/reference_KL": -20.89072608947754, + "fg_logps/reference_chosen": -15.644922256469727, + "fg_logps/reference_rejected": -17.162748336791992, + "fg_loss": 0.8513435125350952, + "fg_rewards/chosen_sum": 0.21025246381759644, + "fg_rewards/rejected_sum": -0.4860546588897705, + "grad_norm": 11.01420885518033, + "kl": 0.0, + "learning_rate": 1.877853881278539e-07, + "logps/chosen": -283.9858993902439, + "logps/rejected": -359.59252303685895, + "loss": 0.4902, + "rewards/chosen": -0.6547578951207603, + "rewards/margins": 1.8769426304075854, + "rewards/rejected": -2.5317005255283456, + "step": 650 + }, + { + "count/fg_chosen": 15.045454978942871, + "count/fg_rejected": 4.222222328186035, + "epoch": 0.6770098730606487, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.071636199951172, + "fg_logps/policy_chosen": -15.461947441101074, + "fg_logps/policy_rejected": -16.1773738861084, + "fg_logps/reference_KL": -20.327823638916016, + "fg_logps/reference_chosen": -14.584894180297852, + "fg_logps/reference_rejected": -15.22111988067627, + "fg_loss": 0.7847881317138672, + "fg_rewards/chosen_sum": -0.37347137928009033, + "fg_rewards/rejected_sum": -0.5809666514396667, + "grad_norm": 10.971292603882963, + "kl": 0.0, + "learning_rate": 1.8207762557077624e-07, + "logps/chosen": -292.3511575838415, + "logps/rejected": -411.4802684294872, + "loss": 0.4927, + "rewards/chosen": -0.615756523318407, + "rewards/margins": 2.001151623466449, + "rewards/rejected": -2.616908146784856, + "step": 660 + }, + { + "count/fg_chosen": 17.521739959716797, + "count/fg_rejected": 3.8947367668151855, + "epoch": 0.6872675984100526, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.02415657043457, + "fg_logps/policy_chosen": -16.135929107666016, + "fg_logps/policy_rejected": -17.662372589111328, + "fg_logps/reference_KL": -20.120943069458008, + "fg_logps/reference_chosen": -15.36439037322998, + "fg_logps/reference_rejected": -16.619319915771484, + "fg_loss": 0.6095057725906372, + "fg_rewards/chosen_sum": -0.9091306924819946, + "fg_rewards/rejected_sum": -0.6099987030029297, + "grad_norm": 11.277192107560657, + "kl": 0.0, + "learning_rate": 1.7636986301369863e-07, + "logps/chosen": -329.480522260274, + "logps/rejected": -394.565149066092, + "loss": 0.5064, + "rewards/chosen": -1.2582495702456122, + "rewards/margins": 1.6947859816805733, + "rewards/rejected": -2.9530355519261855, + "step": 670 + }, + { + "count/fg_chosen": 17.882352828979492, + "count/fg_rejected": 2.9166667461395264, + "epoch": 0.6975253237594563, + "fg_kl": NaN, + "fg_logps/policy_KL": -18.715160369873047, + "fg_logps/policy_chosen": -13.929475784301758, + "fg_logps/policy_rejected": -18.618078231811523, + "fg_logps/reference_KL": -18.27341651916504, + "fg_logps/reference_chosen": -13.522153854370117, + "fg_logps/reference_rejected": -17.502342224121094, + "fg_loss": 0.6171839237213135, + "fg_rewards/chosen_sum": -0.3216637372970581, + "fg_rewards/rejected_sum": -0.12122553586959839, + "grad_norm": 10.871199583632286, + "kl": 0.07263422012329102, + "learning_rate": 1.70662100456621e-07, + "logps/chosen": -290.58342978395063, + "logps/rejected": -358.15182456487344, + "loss": 0.473, + "rewards/chosen": -0.5404636712721836, + "rewards/margins": 2.1079875423230496, + "rewards/rejected": -2.648451213595233, + "step": 680 + }, + { + "count/fg_chosen": 16.10714340209961, + "count/fg_rejected": 3.5714285373687744, + "epoch": 0.7077830491088601, + "fg_kl": NaN, + "fg_logps/policy_KL": -20.88849449157715, + "fg_logps/policy_chosen": -15.352334022521973, + "fg_logps/policy_rejected": -19.385269165039062, + "fg_logps/reference_KL": -19.914424896240234, + "fg_logps/reference_chosen": -14.209139823913574, + "fg_logps/reference_rejected": -18.167560577392578, + "fg_loss": 0.6616089344024658, + "fg_rewards/chosen_sum": -0.9578856229782104, + "fg_rewards/rejected_sum": -0.687567949295044, + "grad_norm": 7.140501610514079, + "kl": 0.0, + "learning_rate": 1.649543378995434e-07, + "logps/chosen": -293.50786458333334, + "logps/rejected": -386.10523897058823, + "loss": 0.4551, + "rewards/chosen": -0.3227819315592448, + "rewards/margins": 2.913227015476601, + "rewards/rejected": -3.2360089470358457, + "step": 690 + }, + { + "count/fg_chosen": 18.0, + "count/fg_rejected": 4.599999904632568, + "epoch": 0.7180407744582639, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.225204467773438, + "fg_logps/policy_chosen": -15.224019050598145, + "fg_logps/policy_rejected": -17.033180236816406, + "fg_logps/reference_KL": -19.995820999145508, + "fg_logps/reference_chosen": -13.913763046264648, + "fg_logps/reference_rejected": -15.521989822387695, + "fg_loss": 0.799098014831543, + "fg_rewards/chosen_sum": -1.0094504356384277, + "fg_rewards/rejected_sum": -0.5853248238563538, + "grad_norm": 12.12648011854002, + "kl": 0.0, + "learning_rate": 1.5924657534246573e-07, + "logps/chosen": -344.3683810763889, + "logps/rejected": -375.64266183035716, + "loss": 0.4812, + "rewards/chosen": -0.6706111907958985, + "rewards/margins": 2.2407723018101287, + "rewards/rejected": -2.911383492606027, + "step": 700 + }, + { + "count/fg_chosen": 17.1200008392334, + "count/fg_rejected": 3.095238208770752, + "epoch": 0.7282984998076677, + "fg_kl": NaN, + "fg_logps/policy_KL": -19.293447494506836, + "fg_logps/policy_chosen": -13.494970321655273, + "fg_logps/policy_rejected": -16.938003540039062, + "fg_logps/reference_KL": -18.482240676879883, + "fg_logps/reference_chosen": -13.644604682922363, + "fg_logps/reference_rejected": -16.76604652404785, + "fg_loss": 0.7007278800010681, + "fg_rewards/chosen_sum": 0.5598518252372742, + "fg_rewards/rejected_sum": -0.21593458950519562, + "grad_norm": 14.686077759756456, + "kl": 0.005865669343620539, + "learning_rate": 1.5353881278538813e-07, + "logps/chosen": -292.7752586570946, + "logps/rejected": -331.10535519622096, + "loss": 0.4337, + "rewards/chosen": -0.09871104601267222, + "rewards/margins": 2.7478330866635634, + "rewards/rejected": -2.8465441326762355, + "step": 710 + }, + { + "count/fg_chosen": 16.35714340209961, + "count/fg_rejected": 3.0, + "epoch": 0.7385562251570714, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.972545623779297, + "fg_logps/policy_chosen": -14.150968551635742, + "fg_logps/policy_rejected": -22.872663497924805, + "fg_logps/reference_KL": -20.97467041015625, + "fg_logps/reference_chosen": -14.420245170593262, + "fg_logps/reference_rejected": -23.34649658203125, + "fg_loss": 0.6548932194709778, + "fg_rewards/chosen_sum": 1.0662188529968262, + "fg_rewards/rejected_sum": 0.059594761580228806, + "grad_norm": 10.578828543316053, + "kl": 0.0, + "learning_rate": 1.478310502283105e-07, + "logps/chosen": -315.2850294237013, + "logps/rejected": -390.93107586596386, + "loss": 0.4322, + "rewards/chosen": -0.855069940740412, + "rewards/margins": 1.7850939864527353, + "rewards/rejected": -2.6401639271931474, + "step": 720 + }, + { + "count/fg_chosen": 18.545454025268555, + "count/fg_rejected": 4.666666507720947, + "epoch": 0.7488139505064751, + "fg_kl": NaN, + "fg_logps/policy_KL": -22.168006896972656, + "fg_logps/policy_chosen": -15.429848670959473, + "fg_logps/policy_rejected": -20.59855079650879, + "fg_logps/reference_KL": -21.434024810791016, + "fg_logps/reference_chosen": -15.218304634094238, + "fg_logps/reference_rejected": -20.059797286987305, + "fg_loss": 0.7126922607421875, + "fg_rewards/chosen_sum": 0.7979265451431274, + "fg_rewards/rejected_sum": -0.11843083798885345, + "grad_norm": 14.792127676836103, + "kl": 0.06886587291955948, + "learning_rate": 1.421232876712329e-07, + "logps/chosen": -287.5471157962329, + "logps/rejected": -396.7852460488506, + "loss": 0.437, + "rewards/chosen": 0.011234314474340987, + "rewards/margins": 3.0592356586508855, + "rewards/rejected": -3.0480013441765443, + "step": 730 + }, + { + "count/fg_chosen": 14.793103218078613, + "count/fg_rejected": 4.148148059844971, + "epoch": 0.759071675855879, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.16313934326172, + "fg_logps/policy_chosen": -15.684663772583008, + "fg_logps/policy_rejected": -19.98002052307129, + "fg_logps/reference_KL": -20.154132843017578, + "fg_logps/reference_chosen": -14.688092231750488, + "fg_logps/reference_rejected": -18.67272186279297, + "fg_loss": 0.8155136704444885, + "fg_rewards/chosen_sum": -0.6768214106559753, + "fg_rewards/rejected_sum": -0.5260854959487915, + "grad_norm": 15.891406104752123, + "kl": 0.0, + "learning_rate": 1.3641552511415523e-07, + "logps/chosen": -287.50723958333333, + "logps/rejected": -408.88382352941176, + "loss": 0.5193, + "rewards/chosen": -0.7413622029622395, + "rewards/margins": 2.1912876562978707, + "rewards/rejected": -2.9326498592601102, + "step": 740 + }, + { + "count/fg_chosen": 17.5, + "count/fg_rejected": 4.222222328186035, + "epoch": 0.7693294012052827, + "fg_kl": NaN, + "fg_logps/policy_KL": -18.54020881652832, + "fg_logps/policy_chosen": -14.070555686950684, + "fg_logps/policy_rejected": -19.403593063354492, + "fg_logps/reference_KL": -17.662343978881836, + "fg_logps/reference_chosen": -13.439438819885254, + "fg_logps/reference_rejected": -18.562532424926758, + "fg_loss": 0.7397133111953735, + "fg_rewards/chosen_sum": -0.7740154266357422, + "fg_rewards/rejected_sum": -0.5391371846199036, + "grad_norm": 10.009694005800096, + "kl": 0.0, + "learning_rate": 1.3070776255707763e-07, + "logps/chosen": -312.1899646577381, + "logps/rejected": -437.6450966282895, + "loss": 0.5177, + "rewards/chosen": -1.1885592142740886, + "rewards/margins": 1.7126708448978893, + "rewards/rejected": -2.901230059171978, + "step": 750 + }, + { + "count/fg_chosen": 15.52173900604248, + "count/fg_rejected": 3.25, + "epoch": 0.7795871265546865, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.690628051757812, + "fg_logps/policy_chosen": -15.920393943786621, + "fg_logps/policy_rejected": -17.173805236816406, + "fg_logps/reference_KL": -20.706315994262695, + "fg_logps/reference_chosen": -15.248817443847656, + "fg_logps/reference_rejected": -16.315380096435547, + "fg_loss": 0.7596006393432617, + "fg_rewards/chosen_sum": -0.31831929087638855, + "fg_rewards/rejected_sum": -0.46696019172668457, + "grad_norm": 11.001152687682147, + "kl": 0.1602586805820465, + "learning_rate": 1.25e-07, + "logps/chosen": -315.33207514044943, + "logps/rejected": -346.9461652728873, + "loss": 0.5177, + "rewards/chosen": -0.877791586886631, + "rewards/margins": 1.9533982805897114, + "rewards/rejected": -2.8311898674763425, + "step": 760 + }, + { + "count/fg_chosen": 13.954545021057129, + "count/fg_rejected": 5.0625, + "epoch": 0.7898448519040903, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.25006866455078, + "fg_logps/policy_chosen": -15.683282852172852, + "fg_logps/policy_rejected": -20.789840698242188, + "fg_logps/reference_KL": -20.35251808166504, + "fg_logps/reference_chosen": -15.138045310974121, + "fg_logps/reference_rejected": -19.545236587524414, + "fg_loss": 0.7877352833747864, + "fg_rewards/chosen_sum": -0.0805804431438446, + "fg_rewards/rejected_sum": -0.919762909412384, + "grad_norm": 8.914441470208311, + "kl": 0.0056018829345703125, + "learning_rate": 1.1929223744292236e-07, + "logps/chosen": -287.3155048076923, + "logps/rejected": -411.3341749237805, + "loss": 0.4771, + "rewards/chosen": -0.5340637304844, + "rewards/margins": 2.177349211052852, + "rewards/rejected": -2.7114129415372523, + "step": 770 + }, + { + "count/fg_chosen": 16.94444465637207, + "count/fg_rejected": 3.8125, + "epoch": 0.800102577253494, + "fg_kl": NaN, + "fg_logps/policy_KL": -19.90068244934082, + "fg_logps/policy_chosen": -14.70145034790039, + "fg_logps/policy_rejected": -18.811277389526367, + "fg_logps/reference_KL": -19.578754425048828, + "fg_logps/reference_chosen": -14.712576866149902, + "fg_logps/reference_rejected": -18.912837982177734, + "fg_loss": 0.7731603980064392, + "fg_rewards/chosen_sum": 0.47520291805267334, + "fg_rewards/rejected_sum": -0.34054040908813477, + "grad_norm": 8.587072014013287, + "kl": 0.09591908752918243, + "learning_rate": 1.1358447488584474e-07, + "logps/chosen": -269.6305419921875, + "logps/rejected": -346.35361328125, + "loss": 0.4703, + "rewards/chosen": -0.6143415451049805, + "rewards/margins": 1.9741443634033204, + "rewards/rejected": -2.588485908508301, + "step": 780 + }, + { + "count/fg_chosen": 16.72222137451172, + "count/fg_rejected": 4.916666507720947, + "epoch": 0.8103603026028978, + "fg_kl": NaN, + "fg_logps/policy_KL": -19.655256271362305, + "fg_logps/policy_chosen": -14.51056957244873, + "fg_logps/policy_rejected": -17.415449142456055, + "fg_logps/reference_KL": -19.09449005126953, + "fg_logps/reference_chosen": -14.248104095458984, + "fg_logps/reference_rejected": -16.654375076293945, + "fg_loss": 0.6147817969322205, + "fg_rewards/chosen_sum": -0.14415740966796875, + "fg_rewards/rejected_sum": -0.5222800970077515, + "grad_norm": 15.457265468907643, + "kl": 0.08856544643640518, + "learning_rate": 1.0787671232876712e-07, + "logps/chosen": -288.4888599537037, + "logps/rejected": -336.5465783227848, + "loss": 0.4963, + "rewards/chosen": -0.8326729668511285, + "rewards/margins": 1.523808464554627, + "rewards/rejected": -2.3564814314057556, + "step": 790 + }, + { + "count/fg_chosen": 18.117647171020508, + "count/fg_rejected": 5.333333492279053, + "epoch": 0.8206180279523015, + "fg_kl": NaN, + "fg_logps/policy_KL": -18.511550903320312, + "fg_logps/policy_chosen": -13.713569641113281, + "fg_logps/policy_rejected": -20.020904541015625, + "fg_logps/reference_KL": -17.690017700195312, + "fg_logps/reference_chosen": -13.43185806274414, + "fg_logps/reference_rejected": -19.19187355041504, + "fg_loss": 0.7455827593803406, + "fg_rewards/chosen_sum": 0.051257286220788956, + "fg_rewards/rejected_sum": -0.5338117480278015, + "grad_norm": 15.017949800951369, + "kl": 0.0, + "learning_rate": 1.0216894977168949e-07, + "logps/chosen": -324.4031684027778, + "logps/rejected": -468.073828125, + "loss": 0.5189, + "rewards/chosen": -0.744783443874783, + "rewards/margins": 1.6178517053997705, + "rewards/rejected": -2.3626351492745536, + "step": 800 + }, + { + "epoch": 0.8206180279523015, + "eval_count/fg_chosen": 16.01369857788086, + "eval_count/fg_rejected": 4.0824174880981445, + "eval_fg_kl": NaN, + "eval_fg_logps/policy_KL": -21.125959396362305, + "eval_fg_logps/policy_chosen": -15.278054237365723, + "eval_fg_logps/policy_rejected": -19.65519905090332, + "eval_fg_logps/reference_KL": -20.207021713256836, + "eval_fg_logps/reference_chosen": -14.929487228393555, + "eval_fg_logps/reference_rejected": -18.78682518005371, + "eval_fg_loss": 0.736534595489502, + "eval_fg_rewards/chosen_sum": 0.0694078877568245, + "eval_fg_rewards/rejected_sum": -0.3622537851333618, + "eval_kl": 0.008128926157951355, + "eval_logps/chosen": -299.7121112089202, + "eval_logps/rejected": -364.37436224489795, + "eval_loss": 0.4815324544906616, + "eval_rewards/chosen": -0.6601278815470951, + "eval_rewards/margins": 1.9833717220703937, + "eval_rewards/rejected": -2.6434996036174887, + "eval_runtime": 808.1531, + "eval_samples_per_second": 2.144, + "eval_steps_per_second": 1.073, + "step": 800 + }, + { + "count/fg_chosen": 16.33333396911621, + "count/fg_rejected": 2.7857143878936768, + "epoch": 0.8308757533017054, + "fg_kl": NaN, + "fg_logps/policy_KL": -23.438562393188477, + "fg_logps/policy_chosen": -16.027084350585938, + "fg_logps/policy_rejected": -18.6650390625, + "fg_logps/reference_KL": -22.405675888061523, + "fg_logps/reference_chosen": -15.913044929504395, + "fg_logps/reference_rejected": -18.558101654052734, + "fg_loss": 0.6452624201774597, + "fg_rewards/chosen_sum": 0.40020328760147095, + "fg_rewards/rejected_sum": -0.3264440894126892, + "grad_norm": 7.011400584597267, + "kl": 0.0, + "learning_rate": 9.646118721461187e-08, + "logps/chosen": -305.2624952936747, + "logps/rejected": -375.5898944805195, + "loss": 0.4673, + "rewards/chosen": -0.7119015613234186, + "rewards/margins": 2.1217946801792196, + "rewards/rejected": -2.833696241502638, + "step": 810 + }, + { + "count/fg_chosen": 17.05555534362793, + "count/fg_rejected": 4.8125, + "epoch": 0.8411334786511091, + "fg_kl": NaN, + "fg_logps/policy_KL": -20.090803146362305, + "fg_logps/policy_chosen": -13.760422706604004, + "fg_logps/policy_rejected": -19.657625198364258, + "fg_logps/reference_KL": -19.27540397644043, + "fg_logps/reference_chosen": -13.429532051086426, + "fg_logps/reference_rejected": -18.345483779907227, + "fg_loss": 0.7558559775352478, + "fg_rewards/chosen_sum": 0.17663703858852386, + "fg_rewards/rejected_sum": -0.7809099555015564, + "grad_norm": 12.43361776512671, + "kl": 0.0, + "learning_rate": 9.075342465753424e-08, + "logps/chosen": -316.12268350290697, + "logps/rejected": -377.34243032094594, + "loss": 0.4807, + "rewards/chosen": -0.8466514764830123, + "rewards/margins": 2.1515591211996314, + "rewards/rejected": -2.9982105976826436, + "step": 820 + }, + { + "count/fg_chosen": 18.95833396911621, + "count/fg_rejected": 3.5999999046325684, + "epoch": 0.8513912040005129, + "fg_kl": NaN, + "fg_logps/policy_KL": -20.093595504760742, + "fg_logps/policy_chosen": -14.394129753112793, + "fg_logps/policy_rejected": -17.22884178161621, + "fg_logps/reference_KL": -19.071619033813477, + "fg_logps/reference_chosen": -13.635478019714355, + "fg_logps/reference_rejected": -16.50658416748047, + "fg_loss": 0.7184759974479675, + "fg_rewards/chosen_sum": 0.43327081203460693, + "fg_rewards/rejected_sum": -0.22198212146759033, + "grad_norm": 13.25277133030139, + "kl": 0.04477081447839737, + "learning_rate": 8.504566210045662e-08, + "logps/chosen": -338.75426793981484, + "logps/rejected": -406.7663419699367, + "loss": 0.5598, + "rewards/chosen": -0.7480648653006848, + "rewards/margins": 1.1518268552417399, + "rewards/rejected": -1.8998917205424248, + "step": 830 + }, + { + "count/fg_chosen": 16.428571701049805, + "count/fg_rejected": 4.277777671813965, + "epoch": 0.8616489293499167, + "fg_kl": NaN, + "fg_logps/policy_KL": -19.675304412841797, + "fg_logps/policy_chosen": -14.73193073272705, + "fg_logps/policy_rejected": -17.31828498840332, + "fg_logps/reference_KL": -19.20144271850586, + "fg_logps/reference_chosen": -14.565011024475098, + "fg_logps/reference_rejected": -17.02730941772461, + "fg_loss": 0.7666863799095154, + "fg_rewards/chosen_sum": 0.5958693623542786, + "fg_rewards/rejected_sum": -0.01988927833735943, + "grad_norm": 11.745424230577617, + "kl": 0.0641721710562706, + "learning_rate": 7.933789954337899e-08, + "logps/chosen": -295.41467126623377, + "logps/rejected": -367.2850856551205, + "loss": 0.5065, + "rewards/chosen": -0.7968547375171215, + "rewards/margins": 1.916948473373315, + "rewards/rejected": -2.7138032108904366, + "step": 840 + }, + { + "count/fg_chosen": 16.1200008392334, + "count/fg_rejected": 4.5714287757873535, + "epoch": 0.8719066546993204, + "fg_kl": NaN, + "fg_logps/policy_KL": -19.93068504333496, + "fg_logps/policy_chosen": -14.386673927307129, + "fg_logps/policy_rejected": -18.88810157775879, + "fg_logps/reference_KL": -19.163719177246094, + "fg_logps/reference_chosen": -13.809516906738281, + "fg_logps/reference_rejected": -17.88907241821289, + "fg_loss": 0.7888970375061035, + "fg_rewards/chosen_sum": -0.009096489287912846, + "fg_rewards/rejected_sum": -0.5189210772514343, + "grad_norm": 10.115452996310198, + "kl": 0.027889441698789597, + "learning_rate": 7.363013698630137e-08, + "logps/chosen": -345.9682221283784, + "logps/rejected": -356.40765806686045, + "loss": 0.5075, + "rewards/chosen": -0.9859014975058066, + "rewards/margins": 1.6604730496984996, + "rewards/rejected": -2.646374547204306, + "step": 850 + }, + { + "count/fg_chosen": 14.9375, + "count/fg_rejected": 4.214285850524902, + "epoch": 0.8821643800487242, + "fg_kl": NaN, + "fg_logps/policy_KL": -22.199317932128906, + "fg_logps/policy_chosen": -14.535076141357422, + "fg_logps/policy_rejected": -18.46771240234375, + "fg_logps/reference_KL": -20.52707862854004, + "fg_logps/reference_chosen": -14.20512866973877, + "fg_logps/reference_rejected": -17.470090866088867, + "fg_loss": 0.8201218247413635, + "fg_rewards/chosen_sum": -0.14333221316337585, + "fg_rewards/rejected_sum": -0.3655402958393097, + "grad_norm": 10.49340621720449, + "kl": 0.0, + "learning_rate": 6.792237442922374e-08, + "logps/chosen": -259.49934050324674, + "logps/rejected": -336.4366999246988, + "loss": 0.4497, + "rewards/chosen": -0.43501192563539975, + "rewards/margins": 2.218185581188772, + "rewards/rejected": -2.6531975068241715, + "step": 860 + }, + { + "count/fg_chosen": 15.117647171020508, + "count/fg_rejected": 3.076923131942749, + "epoch": 0.8924221053981279, + "fg_kl": NaN, + "fg_logps/policy_KL": -22.28684425354004, + "fg_logps/policy_chosen": -14.350448608398438, + "fg_logps/policy_rejected": -20.323551177978516, + "fg_logps/reference_KL": -21.138818740844727, + "fg_logps/reference_chosen": -14.148969650268555, + "fg_logps/reference_rejected": -19.21809959411621, + "fg_loss": 0.7609926462173462, + "fg_rewards/chosen_sum": 0.2540108561515808, + "fg_rewards/rejected_sum": -0.4159541428089142, + "grad_norm": 14.052655899212164, + "kl": 0.0, + "learning_rate": 6.221461187214611e-08, + "logps/chosen": -313.48386548913044, + "logps/rejected": -408.48944024725273, + "loss": 0.4252, + "rewards/chosen": -0.7578884622325068, + "rewards/margins": 2.286565588355577, + "rewards/rejected": -3.044454050588084, + "step": 870 + }, + { + "count/fg_chosen": 13.785714149475098, + "count/fg_rejected": 4.363636493682861, + "epoch": 0.9026798307475318, + "fg_kl": NaN, + "fg_logps/policy_KL": -23.224103927612305, + "fg_logps/policy_chosen": -17.700504302978516, + "fg_logps/policy_rejected": -18.443458557128906, + "fg_logps/reference_KL": -21.812795639038086, + "fg_logps/reference_chosen": -16.529930114746094, + "fg_logps/reference_rejected": -16.90056800842285, + "fg_loss": 0.861905038356781, + "fg_rewards/chosen_sum": -1.0540215969085693, + "fg_rewards/rejected_sum": -0.6782628297805786, + "grad_norm": 10.169312742642402, + "kl": 0.0, + "learning_rate": 5.650684931506849e-08, + "logps/chosen": -271.11566840277777, + "logps/rejected": -324.0809215198864, + "loss": 0.4212, + "rewards/chosen": -0.5925836563110352, + "rewards/margins": 1.9108110774647105, + "rewards/rejected": -2.5033947337757456, + "step": 880 + }, + { + "count/fg_chosen": 17.526315689086914, + "count/fg_rejected": 6.0, + "epoch": 0.9129375560969355, + "fg_kl": NaN, + "fg_logps/policy_KL": -19.294906616210938, + "fg_logps/policy_chosen": -13.935691833496094, + "fg_logps/policy_rejected": -16.94110107421875, + "fg_logps/reference_KL": -18.384138107299805, + "fg_logps/reference_chosen": -13.67742919921875, + "fg_logps/reference_rejected": -15.758844375610352, + "fg_loss": 0.6794268488883972, + "fg_rewards/chosen_sum": 0.0076361955143511295, + "fg_rewards/rejected_sum": -0.4533938765525818, + "grad_norm": 9.707055272049164, + "kl": 0.0, + "learning_rate": 5.0799086757990863e-08, + "logps/chosen": -275.1984604779412, + "logps/rejected": -400.91375, + "loss": 0.4898, + "rewards/chosen": -0.607273774988511, + "rewards/margins": 2.4971288942823224, + "rewards/rejected": -3.1044026692708333, + "step": 890 + }, + { + "count/fg_chosen": 16.18181800842285, + "count/fg_rejected": 5.333333492279053, + "epoch": 0.9231952814463392, + "fg_kl": NaN, + "fg_logps/policy_KL": -19.024534225463867, + "fg_logps/policy_chosen": -14.489638328552246, + "fg_logps/policy_rejected": -17.007856369018555, + "fg_logps/reference_KL": -18.624956130981445, + "fg_logps/reference_chosen": -14.467068672180176, + "fg_logps/reference_rejected": -16.36551284790039, + "fg_loss": 0.7552723288536072, + "fg_rewards/chosen_sum": 0.37475159764289856, + "fg_rewards/rejected_sum": -0.4549465775489807, + "grad_norm": 12.023866951403578, + "kl": 0.0394529327750206, + "learning_rate": 4.509132420091324e-08, + "logps/chosen": -344.09639537183546, + "logps/rejected": -425.6873553240741, + "loss": 0.4817, + "rewards/chosen": -0.8015480524376978, + "rewards/margins": 2.245464465043977, + "rewards/rejected": -3.0470125174816745, + "step": 900 + }, + { + "count/fg_chosen": 16.54166603088379, + "count/fg_rejected": 4.318181991577148, + "epoch": 0.9334530067957431, + "fg_kl": NaN, + "fg_logps/policy_KL": -20.088489532470703, + "fg_logps/policy_chosen": -15.405600547790527, + "fg_logps/policy_rejected": -16.747190475463867, + "fg_logps/reference_KL": -19.137371063232422, + "fg_logps/reference_chosen": -14.657370567321777, + "fg_logps/reference_rejected": -15.770890235900879, + "fg_loss": 0.6983441114425659, + "fg_rewards/chosen_sum": -0.3915860652923584, + "fg_rewards/rejected_sum": -0.3197772204875946, + "grad_norm": 11.088479553468312, + "kl": 0.0, + "learning_rate": 3.938356164383561e-08, + "logps/chosen": -306.00605715981015, + "logps/rejected": -396.05049189814815, + "loss": 0.4692, + "rewards/chosen": -0.3026618957519531, + "rewards/margins": 2.5836239567509405, + "rewards/rejected": -2.8862858525028936, + "step": 910 + }, + { + "count/fg_chosen": 17.454545974731445, + "count/fg_rejected": 3.4000000953674316, + "epoch": 0.9437107321451468, + "fg_kl": NaN, + "fg_logps/policy_KL": -24.18083953857422, + "fg_logps/policy_chosen": -15.12738037109375, + "fg_logps/policy_rejected": -22.837459564208984, + "fg_logps/reference_KL": -22.198238372802734, + "fg_logps/reference_chosen": -13.988216400146484, + "fg_logps/reference_rejected": -20.653087615966797, + "fg_loss": 0.7821993231773376, + "fg_rewards/chosen_sum": -1.5599383115768433, + "fg_rewards/rejected_sum": -0.6531942486763, + "grad_norm": 13.573243756082668, + "kl": 0.0, + "learning_rate": 3.367579908675799e-08, + "logps/chosen": -334.25313527960526, + "logps/rejected": -345.7750651041667, + "loss": 0.5174, + "rewards/chosen": -1.169078224583676, + "rewards/margins": 1.8482150696871575, + "rewards/rejected": -3.0172932942708335, + "step": 920 + }, + { + "count/fg_chosen": 14.75, + "count/fg_rejected": 3.5, + "epoch": 0.9539684574945506, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.988378524780273, + "fg_logps/policy_chosen": -15.057655334472656, + "fg_logps/policy_rejected": -21.019062042236328, + "fg_logps/reference_KL": -20.459566116333008, + "fg_logps/reference_chosen": -14.091443061828613, + "fg_logps/reference_rejected": -20.186948776245117, + "fg_loss": 0.7185892462730408, + "fg_rewards/chosen_sum": -0.44822633266448975, + "fg_rewards/rejected_sum": -0.26188215613365173, + "grad_norm": 16.393627175860047, + "kl": 0.0, + "learning_rate": 2.796803652968036e-08, + "logps/chosen": -283.63548677884614, + "logps/rejected": -336.15740131578946, + "loss": 0.4039, + "rewards/chosen": -0.3187708928034856, + "rewards/margins": 2.802559930592896, + "rewards/rejected": -3.1213308233963817, + "step": 930 + }, + { + "count/fg_chosen": 18.83333396911621, + "count/fg_rejected": 5.933333396911621, + "epoch": 0.9642261828439543, + "fg_kl": NaN, + "fg_logps/policy_KL": -20.48985481262207, + "fg_logps/policy_chosen": -15.466116905212402, + "fg_logps/policy_rejected": -21.363821029663086, + "fg_logps/reference_KL": -19.345491409301758, + "fg_logps/reference_chosen": -14.387165069580078, + "fg_logps/reference_rejected": -19.8946533203125, + "fg_loss": 0.7254918217658997, + "fg_rewards/chosen_sum": -1.1295708417892456, + "fg_rewards/rejected_sum": -0.7322808504104614, + "grad_norm": 12.073302615788874, + "kl": 0.0, + "learning_rate": 2.2260273972602736e-08, + "logps/chosen": -311.50599500868054, + "logps/rejected": -380.01899857954544, + "loss": 0.4599, + "rewards/chosen": -0.8199851247999403, + "rewards/margins": 2.1154369055622757, + "rewards/rejected": -2.935422030362216, + "step": 940 + }, + { + "count/fg_chosen": 15.636363983154297, + "count/fg_rejected": 3.4000000953674316, + "epoch": 0.9744839081933582, + "fg_kl": NaN, + "fg_logps/policy_KL": -18.018638610839844, + "fg_logps/policy_chosen": -14.090137481689453, + "fg_logps/policy_rejected": -13.102763175964355, + "fg_logps/reference_KL": -16.83769416809082, + "fg_logps/reference_chosen": -13.336016654968262, + "fg_logps/reference_rejected": -12.24677848815918, + "fg_loss": 0.8313923478126526, + "fg_rewards/chosen_sum": -0.822435736656189, + "fg_rewards/rejected_sum": -0.4055534303188324, + "grad_norm": 8.942456920046205, + "kl": 0.027703475207090378, + "learning_rate": 1.6552511415525114e-08, + "logps/chosen": -319.5060622970779, + "logps/rejected": -353.5703830948795, + "loss": 0.4562, + "rewards/chosen": -1.2102871188869724, + "rewards/margins": 1.2892790167493229, + "rewards/rejected": -2.4995661356362953, + "step": 950 + }, + { + "count/fg_chosen": 14.909090995788574, + "count/fg_rejected": 3.6315789222717285, + "epoch": 0.9847416335427619, + "fg_kl": NaN, + "fg_logps/policy_KL": -21.546527862548828, + "fg_logps/policy_chosen": -15.540050506591797, + "fg_logps/policy_rejected": -20.36016273498535, + "fg_logps/reference_KL": -20.43901252746582, + "fg_logps/reference_chosen": -14.649651527404785, + "fg_logps/reference_rejected": -18.264400482177734, + "fg_loss": 0.7249688506126404, + "fg_rewards/chosen_sum": -0.600669801235199, + "fg_rewards/rejected_sum": -0.7846862077713013, + "grad_norm": 13.343937741670935, + "kl": 0.0, + "learning_rate": 1.0844748858447487e-08, + "logps/chosen": -326.93775699013156, + "logps/rejected": -363.2979445684524, + "loss": 0.5157, + "rewards/chosen": -1.1762562801963405, + "rewards/margins": 1.8776036121492694, + "rewards/rejected": -3.05385989234561, + "step": 960 + }, + { + "count/fg_chosen": 14.61111068725586, + "count/fg_rejected": 3.3333332538604736, + "epoch": 0.9949993588921656, + "fg_kl": NaN, + "fg_logps/policy_KL": -24.816984176635742, + "fg_logps/policy_chosen": -17.12736701965332, + "fg_logps/policy_rejected": -22.242828369140625, + "fg_logps/reference_KL": -22.654382705688477, + "fg_logps/reference_chosen": -15.033406257629395, + "fg_logps/reference_rejected": -21.171716690063477, + "fg_loss": 0.7910918593406677, + "fg_rewards/chosen_sum": -1.5589678287506104, + "fg_rewards/rejected_sum": -0.5098855495452881, + "grad_norm": 12.686683889493278, + "kl": 0.0, + "learning_rate": 5.136986301369862e-09, + "logps/chosen": -375.69940350506755, + "logps/rejected": -364.8770212572674, + "loss": 0.481, + "rewards/chosen": -1.2619872222075592, + "rewards/margins": 1.2282116672364813, + "rewards/rejected": -2.4901988894440406, + "step": 970 + }, + { + "epoch": 0.9991024490319271, + "step": 974, + "total_flos": 0.0, + "train_loss": 0.5029905087159644, + "train_runtime": 13638.2416, + "train_samples_per_second": 1.144, + "train_steps_per_second": 0.071 + } + ], + "logging_steps": 10, + "max_steps": 974, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 500, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": false, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +}