| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 400, | |
| "global_step": 975, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "count/fg_chosen": 30.85714340209961, | |
| "count/fg_rejected": 7.4285712242126465, | |
| "epoch": 0.010256410256410256, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -11.948674201965332, | |
| "fg_logps/policy_chosen": -6.262445449829102, | |
| "fg_logps/policy_rejected": -8.74467945098877, | |
| "fg_logps/reference_KL": -11.94157600402832, | |
| "fg_logps/reference_chosen": -6.2594828605651855, | |
| "fg_logps/reference_rejected": -8.742448806762695, | |
| "fg_loss": 0.8008173704147339, | |
| "fg_rewards/chosen_sum": -0.008917576633393764, | |
| "fg_rewards/rejected_sum": -0.0010543327080085874, | |
| "grad_norm": 70.97090228694296, | |
| "kl": 0.15787295997142792, | |
| "learning_rate": 2.5110157309792834e-07, | |
| "logps/chosen": -366.76351768092104, | |
| "logps/rejected": -369.69268508184524, | |
| "loss": 0.6347, | |
| "rewards/chosen": 0.014076207813463713, | |
| "rewards/margins": -0.000743936476552097, | |
| "rewards/rejected": 0.01482014429001581, | |
| "step": 10 | |
| }, | |
| { | |
| "count/fg_chosen": 26.352941513061523, | |
| "count/fg_rejected": 6.058823585510254, | |
| "epoch": 0.020512820512820513, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -10.825294494628906, | |
| "fg_logps/policy_chosen": -5.95189905166626, | |
| "fg_logps/policy_rejected": -5.48292350769043, | |
| "fg_logps/reference_KL": -10.80107307434082, | |
| "fg_logps/reference_chosen": -5.9293718338012695, | |
| "fg_logps/reference_rejected": -5.445353984832764, | |
| "fg_loss": 0.7557108402252197, | |
| "fg_rewards/chosen_sum": -0.05455803498625755, | |
| "fg_rewards/rejected_sum": -0.025491168722510338, | |
| "grad_norm": 76.27540979070403, | |
| "kl": 0.05270981788635254, | |
| "learning_rate": 3.2669067855881653e-07, | |
| "logps/chosen": -385.705078125, | |
| "logps/rejected": -347.460890436747, | |
| "loss": 0.5881, | |
| "rewards/chosen": 0.02315927016270625, | |
| "rewards/margins": 0.05135368041786262, | |
| "rewards/rejected": -0.028194410255156368, | |
| "step": 20 | |
| }, | |
| { | |
| "count/fg_chosen": 27.0, | |
| "count/fg_rejected": 7.176470756530762, | |
| "epoch": 0.03076923076923077, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.925114631652832, | |
| "fg_logps/policy_chosen": -6.261083602905273, | |
| "fg_logps/policy_rejected": -7.534660816192627, | |
| "fg_logps/reference_KL": -12.83480167388916, | |
| "fg_logps/reference_chosen": -6.232151985168457, | |
| "fg_logps/reference_rejected": -7.494457244873047, | |
| "fg_loss": 0.8402522206306458, | |
| "fg_rewards/chosen_sum": -0.0777626782655716, | |
| "fg_rewards/rejected_sum": -0.03122476302087307, | |
| "grad_norm": 112.80411347358924, | |
| "kl": 0.02636871300637722, | |
| "learning_rate": 3.709074707164929e-07, | |
| "logps/chosen": -380.789990234375, | |
| "logps/rejected": -344.6407958984375, | |
| "loss": 0.5949, | |
| "rewards/chosen": 0.017523756623268126, | |
| "rewards/margins": 0.0999738484621048, | |
| "rewards/rejected": -0.08245009183883667, | |
| "step": 30 | |
| }, | |
| { | |
| "count/fg_chosen": 32.60869598388672, | |
| "count/fg_rejected": 8.136363983154297, | |
| "epoch": 0.041025641025641026, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -11.220458030700684, | |
| "fg_logps/policy_chosen": -5.84191370010376, | |
| "fg_logps/policy_rejected": -6.710254192352295, | |
| "fg_logps/reference_KL": -11.054718971252441, | |
| "fg_logps/reference_chosen": -5.7772111892700195, | |
| "fg_logps/reference_rejected": -6.645753860473633, | |
| "fg_loss": 0.808053195476532, | |
| "fg_rewards/chosen_sum": -0.1892387568950653, | |
| "fg_rewards/rejected_sum": -0.05010434612631798, | |
| "grad_norm": 50.84808526748507, | |
| "kl": 0.0, | |
| "learning_rate": 4.022797840197047e-07, | |
| "logps/chosen": -383.87660435267856, | |
| "logps/rejected": -374.25223581414474, | |
| "loss": 0.6121, | |
| "rewards/chosen": 0.04867589473724365, | |
| "rewards/margins": 0.2174466848373413, | |
| "rewards/rejected": -0.16877079010009766, | |
| "step": 40 | |
| }, | |
| { | |
| "count/fg_chosen": 32.79999923706055, | |
| "count/fg_rejected": 9.533333778381348, | |
| "epoch": 0.05128205128205128, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -10.373883247375488, | |
| "fg_logps/policy_chosen": -5.664600849151611, | |
| "fg_logps/policy_rejected": -5.962148189544678, | |
| "fg_logps/reference_KL": -10.145182609558105, | |
| "fg_logps/reference_chosen": -5.588218688964844, | |
| "fg_logps/reference_rejected": -5.897520065307617, | |
| "fg_loss": 0.8677656054496765, | |
| "fg_rewards/chosen_sum": -0.25619086623191833, | |
| "fg_rewards/rejected_sum": -0.07288213074207306, | |
| "grad_norm": 64.57457570643511, | |
| "kl": 0.009938049130141735, | |
| "learning_rate": 4.2661404073496845e-07, | |
| "logps/chosen": -346.134577371988, | |
| "logps/rejected": -368.41335227272725, | |
| "loss": 0.5682, | |
| "rewards/chosen": 0.2387204227677311, | |
| "rewards/margins": 0.3148248284061905, | |
| "rewards/rejected": -0.07610440563845944, | |
| "step": 50 | |
| }, | |
| { | |
| "count/fg_chosen": 26.294116973876953, | |
| "count/fg_rejected": 5.470588207244873, | |
| "epoch": 0.06153846153846154, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.107590675354004, | |
| "fg_logps/policy_chosen": -6.114685535430908, | |
| "fg_logps/policy_rejected": -7.69861364364624, | |
| "fg_logps/reference_KL": -11.640838623046875, | |
| "fg_logps/reference_chosen": -6.00569486618042, | |
| "fg_logps/reference_rejected": -7.617056369781494, | |
| "fg_loss": 0.7082093954086304, | |
| "fg_rewards/chosen_sum": -0.27744388580322266, | |
| "fg_rewards/rejected_sum": -0.06493094563484192, | |
| "grad_norm": 49.41694776530553, | |
| "kl": 0.0, | |
| "learning_rate": 4.4649657617738114e-07, | |
| "logps/chosen": -353.69510690789474, | |
| "logps/rejected": -366.0445033482143, | |
| "loss": 0.5548, | |
| "rewards/chosen": 0.17124160967375102, | |
| "rewards/margins": 0.3800723295761529, | |
| "rewards/rejected": -0.20883071990240187, | |
| "step": 60 | |
| }, | |
| { | |
| "count/fg_chosen": 32.19047546386719, | |
| "count/fg_rejected": 6.238095283508301, | |
| "epoch": 0.07179487179487179, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.570515632629395, | |
| "fg_logps/policy_chosen": -6.555771827697754, | |
| "fg_logps/policy_rejected": -8.589411735534668, | |
| "fg_logps/reference_KL": -12.056818008422852, | |
| "fg_logps/reference_chosen": -6.406361103057861, | |
| "fg_logps/reference_rejected": -8.353743553161621, | |
| "fg_loss": 0.716395378112793, | |
| "fg_rewards/chosen_sum": -0.46484148502349854, | |
| "fg_rewards/rejected_sum": -0.1304880827665329, | |
| "grad_norm": 68.38293723456421, | |
| "kl": 0.0, | |
| "learning_rate": 4.633070203674842e-07, | |
| "logps/chosen": -319.37548828125, | |
| "logps/rejected": -425.4502418154762, | |
| "loss": 0.6056, | |
| "rewards/chosen": -0.016339432252080816, | |
| "rewards/margins": 0.2762198054551481, | |
| "rewards/rejected": -0.2925592377072289, | |
| "step": 70 | |
| }, | |
| { | |
| "count/fg_chosen": 34.77777862548828, | |
| "count/fg_rejected": 9.470588684082031, | |
| "epoch": 0.08205128205128205, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -10.578932762145996, | |
| "fg_logps/policy_chosen": -5.850771427154541, | |
| "fg_logps/policy_rejected": -7.156460762023926, | |
| "fg_logps/reference_KL": -10.092942237854004, | |
| "fg_logps/reference_chosen": -5.712654113769531, | |
| "fg_logps/reference_rejected": -6.944825172424316, | |
| "fg_loss": 0.8019319176673889, | |
| "fg_rewards/chosen_sum": -0.4671042263507843, | |
| "fg_rewards/rejected_sum": -0.17894208431243896, | |
| "grad_norm": 39.81642138358546, | |
| "kl": 0.0, | |
| "learning_rate": 4.77868889480593e-07, | |
| "logps/chosen": -317.1737351190476, | |
| "logps/rejected": -377.7265625, | |
| "loss": 0.5778, | |
| "rewards/chosen": 0.11412754512968518, | |
| "rewards/margins": 0.5135003988605394, | |
| "rewards/rejected": -0.39937285373085424, | |
| "step": 80 | |
| }, | |
| { | |
| "count/fg_chosen": 31.0625, | |
| "count/fg_rejected": 6.25, | |
| "epoch": 0.09230769230769231, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -11.664382934570312, | |
| "fg_logps/policy_chosen": -6.444965839385986, | |
| "fg_logps/policy_rejected": -8.933675765991211, | |
| "fg_logps/reference_KL": -10.985946655273438, | |
| "fg_logps/reference_chosen": -6.227105140686035, | |
| "fg_logps/reference_rejected": -8.714274406433105, | |
| "fg_loss": 0.6460863351821899, | |
| "fg_rewards/chosen_sum": -0.5869948267936707, | |
| "fg_rewards/rejected_sum": -0.14423823356628418, | |
| "grad_norm": 55.62438439854401, | |
| "kl": 0.0, | |
| "learning_rate": 4.907133683350575e-07, | |
| "logps/chosen": -404.47572544642856, | |
| "logps/rejected": -416.0439453125, | |
| "loss": 0.5292, | |
| "rewards/chosen": 0.30132850011189777, | |
| "rewards/margins": 0.755986305705288, | |
| "rewards/rejected": -0.4546578055933902, | |
| "step": 90 | |
| }, | |
| { | |
| "count/fg_chosen": 28.772727966308594, | |
| "count/fg_rejected": 6.333333492279053, | |
| "epoch": 0.10256410256410256, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.864214897155762, | |
| "fg_logps/policy_chosen": -6.919012546539307, | |
| "fg_logps/policy_rejected": -9.152473449707031, | |
| "fg_logps/reference_KL": -12.052042007446289, | |
| "fg_logps/reference_chosen": -6.679240703582764, | |
| "fg_logps/reference_rejected": -8.906697273254395, | |
| "fg_loss": 0.7610839605331421, | |
| "fg_rewards/chosen_sum": -0.674491822719574, | |
| "fg_rewards/rejected_sum": -0.1936338096857071, | |
| "grad_norm": 63.5056444865937, | |
| "kl": 0.0, | |
| "learning_rate": 4.994298745724059e-07, | |
| "logps/chosen": -340.2066359747024, | |
| "logps/rejected": -390.9894377055921, | |
| "loss": 0.6206, | |
| "rewards/chosen": 0.13452401615324475, | |
| "rewards/margins": 0.2098735791997503, | |
| "rewards/rejected": -0.07534956304650557, | |
| "step": 100 | |
| }, | |
| { | |
| "count/fg_chosen": 32.42856979370117, | |
| "count/fg_rejected": 5.736842155456543, | |
| "epoch": 0.11282051282051282, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.667691230773926, | |
| "fg_logps/policy_chosen": -6.325592041015625, | |
| "fg_logps/policy_rejected": -7.5075507164001465, | |
| "fg_logps/reference_KL": -11.706281661987305, | |
| "fg_logps/reference_chosen": -6.0957818031311035, | |
| "fg_logps/reference_rejected": -7.153979778289795, | |
| "fg_loss": 0.6360421180725098, | |
| "fg_rewards/chosen_sum": -0.7651198506355286, | |
| "fg_rewards/rejected_sum": -0.1911022961139679, | |
| "grad_norm": 45.1771736885249, | |
| "kl": 0.0, | |
| "learning_rate": 4.937286202964652e-07, | |
| "logps/chosen": -358.7214664152299, | |
| "logps/rejected": -373.4951305650685, | |
| "loss": 0.5458, | |
| "rewards/chosen": 0.35928555192618533, | |
| "rewards/margins": 0.8201791045234402, | |
| "rewards/rejected": -0.4608935525972549, | |
| "step": 110 | |
| }, | |
| { | |
| "count/fg_chosen": 36.45000076293945, | |
| "count/fg_rejected": 10.210526466369629, | |
| "epoch": 0.12307692307692308, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.05976676940918, | |
| "fg_logps/policy_chosen": -6.244026184082031, | |
| "fg_logps/policy_rejected": -7.9034271240234375, | |
| "fg_logps/reference_KL": -11.134529113769531, | |
| "fg_logps/reference_chosen": -6.0136542320251465, | |
| "fg_logps/reference_rejected": -7.628241062164307, | |
| "fg_loss": 0.881417453289032, | |
| "fg_rewards/chosen_sum": -0.7559553384780884, | |
| "fg_rewards/rejected_sum": -0.3012525737285614, | |
| "grad_norm": 50.09822696941802, | |
| "kl": 0.015017986297607422, | |
| "learning_rate": 4.880273660205244e-07, | |
| "logps/chosen": -320.32579210069446, | |
| "logps/rejected": -356.7809392755682, | |
| "loss": 0.601, | |
| "rewards/chosen": 0.6363146040174696, | |
| "rewards/margins": 0.7909924068836252, | |
| "rewards/rejected": -0.15467780286615546, | |
| "step": 120 | |
| }, | |
| { | |
| "count/fg_chosen": 36.400001525878906, | |
| "count/fg_rejected": 7.933333396911621, | |
| "epoch": 0.13333333333333333, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -11.468809127807617, | |
| "fg_logps/policy_chosen": -6.23488712310791, | |
| "fg_logps/policy_rejected": -6.54256010055542, | |
| "fg_logps/reference_KL": -10.525192260742188, | |
| "fg_logps/reference_chosen": -6.026561260223389, | |
| "fg_logps/reference_rejected": -6.267870903015137, | |
| "fg_loss": 0.8011055588722229, | |
| "fg_rewards/chosen_sum": -0.647752583026886, | |
| "fg_rewards/rejected_sum": -0.20049738883972168, | |
| "grad_norm": 50.115512333731104, | |
| "kl": 0.03811788558959961, | |
| "learning_rate": 4.823261117445838e-07, | |
| "logps/chosen": -450.6170099431818, | |
| "logps/rejected": -392.42621527777777, | |
| "loss": 0.5078, | |
| "rewards/chosen": 1.0679140090942383, | |
| "rewards/margins": 1.1960734128952026, | |
| "rewards/rejected": -0.12815940380096436, | |
| "step": 130 | |
| }, | |
| { | |
| "count/fg_chosen": 31.4761905670166, | |
| "count/fg_rejected": 8.699999809265137, | |
| "epoch": 0.14358974358974358, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.193472862243652, | |
| "fg_logps/policy_chosen": -6.388846397399902, | |
| "fg_logps/policy_rejected": -7.679136753082275, | |
| "fg_logps/reference_KL": -11.09471607208252, | |
| "fg_logps/reference_chosen": -6.169702529907227, | |
| "fg_logps/reference_rejected": -7.332272529602051, | |
| "fg_loss": 0.7396747469902039, | |
| "fg_rewards/chosen_sum": -0.5886417627334595, | |
| "fg_rewards/rejected_sum": -0.24698862433433533, | |
| "grad_norm": 48.94452126464587, | |
| "kl": 0.03351273387670517, | |
| "learning_rate": 4.766248574686431e-07, | |
| "logps/chosen": -334.752628279321, | |
| "logps/rejected": -386.6670292721519, | |
| "loss": 0.5475, | |
| "rewards/chosen": 0.8302505869924286, | |
| "rewards/margins": 1.2393360176688526, | |
| "rewards/rejected": -0.40908543067642406, | |
| "step": 140 | |
| }, | |
| { | |
| "count/fg_chosen": 26.105262756347656, | |
| "count/fg_rejected": 4.526315689086914, | |
| "epoch": 0.15384615384615385, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.575007438659668, | |
| "fg_logps/policy_chosen": -7.0048089027404785, | |
| "fg_logps/policy_rejected": -8.861261367797852, | |
| "fg_logps/reference_KL": -12.408534049987793, | |
| "fg_logps/reference_chosen": -6.816770076751709, | |
| "fg_logps/reference_rejected": -8.450145721435547, | |
| "fg_loss": 0.7104328870773315, | |
| "fg_rewards/chosen_sum": -0.4196644723415375, | |
| "fg_rewards/rejected_sum": -0.17191696166992188, | |
| "grad_norm": 49.69486730354108, | |
| "kl": 0.05726609379053116, | |
| "learning_rate": 4.7092360319270236e-07, | |
| "logps/chosen": -418.9573688271605, | |
| "logps/rejected": -390.3833069620253, | |
| "loss": 0.5064, | |
| "rewards/chosen": 0.9700225076557677, | |
| "rewards/margins": 1.2821282176491542, | |
| "rewards/rejected": -0.31210570999338655, | |
| "step": 150 | |
| }, | |
| { | |
| "count/fg_chosen": 25.764705657958984, | |
| "count/fg_rejected": 4.125, | |
| "epoch": 0.1641025641025641, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -14.1893892288208, | |
| "fg_logps/policy_chosen": -6.999147415161133, | |
| "fg_logps/policy_rejected": -7.968658447265625, | |
| "fg_logps/reference_KL": -12.764519691467285, | |
| "fg_logps/reference_chosen": -6.616185188293457, | |
| "fg_logps/reference_rejected": -7.533709526062012, | |
| "fg_loss": 0.7398098707199097, | |
| "fg_rewards/chosen_sum": -0.8501734733581543, | |
| "fg_rewards/rejected_sum": -0.15099674463272095, | |
| "grad_norm": 41.227539280479604, | |
| "kl": 0.0, | |
| "learning_rate": 4.652223489167617e-07, | |
| "logps/chosen": -377.22486787683823, | |
| "logps/rejected": -458.93584408967394, | |
| "loss": 0.4883, | |
| "rewards/chosen": 0.8492268955006319, | |
| "rewards/margins": 1.2064810067491458, | |
| "rewards/rejected": -0.3572541112485139, | |
| "step": 160 | |
| }, | |
| { | |
| "count/fg_chosen": 33.06666564941406, | |
| "count/fg_rejected": 7.615384578704834, | |
| "epoch": 0.17435897435897435, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.796510696411133, | |
| "fg_logps/policy_chosen": -6.570502758026123, | |
| "fg_logps/policy_rejected": -7.280531883239746, | |
| "fg_logps/reference_KL": -11.486601829528809, | |
| "fg_logps/reference_chosen": -6.288327693939209, | |
| "fg_logps/reference_rejected": -6.919613361358643, | |
| "fg_loss": 0.7529634237289429, | |
| "fg_rewards/chosen_sum": -0.7710135579109192, | |
| "fg_rewards/rejected_sum": -0.2511799931526184, | |
| "grad_norm": 50.55336505699433, | |
| "kl": 0.0, | |
| "learning_rate": 4.5952109464082095e-07, | |
| "logps/chosen": -339.1629430259146, | |
| "logps/rejected": -407.0223607772436, | |
| "loss": 0.4934, | |
| "rewards/chosen": 0.810060268495141, | |
| "rewards/margins": 1.2479861452103855, | |
| "rewards/rejected": -0.4379258767152444, | |
| "step": 170 | |
| }, | |
| { | |
| "count/fg_chosen": 32.875, | |
| "count/fg_rejected": 8.25, | |
| "epoch": 0.18461538461538463, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.054594993591309, | |
| "fg_logps/policy_chosen": -6.169870853424072, | |
| "fg_logps/policy_rejected": -7.339913368225098, | |
| "fg_logps/reference_KL": -10.784111022949219, | |
| "fg_logps/reference_chosen": -5.852520942687988, | |
| "fg_logps/reference_rejected": -6.8034443855285645, | |
| "fg_loss": 0.7260686755180359, | |
| "fg_rewards/chosen_sum": -0.8828132152557373, | |
| "fg_rewards/rejected_sum": -0.5087793469429016, | |
| "grad_norm": 79.2618606536439, | |
| "kl": 0.0, | |
| "learning_rate": 4.5381984036488027e-07, | |
| "logps/chosen": -298.9371427210366, | |
| "logps/rejected": -415.6966145833333, | |
| "loss": 0.5143, | |
| "rewards/chosen": 0.3949350496617759, | |
| "rewards/margins": 1.4249423386679356, | |
| "rewards/rejected": -1.0300072890061598, | |
| "step": 180 | |
| }, | |
| { | |
| "count/fg_chosen": 31.363636016845703, | |
| "count/fg_rejected": 6.409090995788574, | |
| "epoch": 0.19487179487179487, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -11.836668968200684, | |
| "fg_logps/policy_chosen": -6.473412990570068, | |
| "fg_logps/policy_rejected": -7.306280612945557, | |
| "fg_logps/reference_KL": -10.574777603149414, | |
| "fg_logps/reference_chosen": -6.141035079956055, | |
| "fg_logps/reference_rejected": -7.045315265655518, | |
| "fg_loss": 0.7152173519134521, | |
| "fg_rewards/chosen_sum": -0.9164342880249023, | |
| "fg_rewards/rejected_sum": -0.21355971693992615, | |
| "grad_norm": 62.51457716342411, | |
| "kl": 0.0, | |
| "learning_rate": 4.4811858608893954e-07, | |
| "logps/chosen": -378.7162252286585, | |
| "logps/rejected": -380.27271133814105, | |
| "loss": 0.482, | |
| "rewards/chosen": 0.9614362949278297, | |
| "rewards/margins": 1.7487119516035108, | |
| "rewards/rejected": -0.7872756566756811, | |
| "step": 190 | |
| }, | |
| { | |
| "count/fg_chosen": 27.789474487304688, | |
| "count/fg_rejected": 6.263157844543457, | |
| "epoch": 0.20512820512820512, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -14.003227233886719, | |
| "fg_logps/policy_chosen": -6.706553936004639, | |
| "fg_logps/policy_rejected": -7.728447437286377, | |
| "fg_logps/reference_KL": -12.440956115722656, | |
| "fg_logps/reference_chosen": -6.388577938079834, | |
| "fg_logps/reference_rejected": -7.369418621063232, | |
| "fg_loss": 0.7191720008850098, | |
| "fg_rewards/chosen_sum": -0.8144214749336243, | |
| "fg_rewards/rejected_sum": -0.220667764544487, | |
| "grad_norm": 53.07709071243361, | |
| "kl": 0.016681909561157227, | |
| "learning_rate": 4.4241733181299887e-07, | |
| "logps/chosen": -376.6458753360215, | |
| "logps/rejected": -420.5650652985075, | |
| "loss": 0.5059, | |
| "rewards/chosen": 0.6015197179650748, | |
| "rewards/margins": 1.3459980290767528, | |
| "rewards/rejected": -0.744478311111678, | |
| "step": 200 | |
| }, | |
| { | |
| "count/fg_chosen": 35.38461685180664, | |
| "count/fg_rejected": 9.230769157409668, | |
| "epoch": 0.2153846153846154, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.624795913696289, | |
| "fg_logps/policy_chosen": -6.518624305725098, | |
| "fg_logps/policy_rejected": -6.696259021759033, | |
| "fg_logps/reference_KL": -12.152335166931152, | |
| "fg_logps/reference_chosen": -6.259435653686523, | |
| "fg_logps/reference_rejected": -6.192153453826904, | |
| "fg_loss": 0.7262544631958008, | |
| "fg_rewards/chosen_sum": -0.6904063820838928, | |
| "fg_rewards/rejected_sum": -0.5146002173423767, | |
| "grad_norm": 53.74268239721948, | |
| "kl": 0.0, | |
| "learning_rate": 4.3671607753705814e-07, | |
| "logps/chosen": -386.4969911317568, | |
| "logps/rejected": -467.1247274709302, | |
| "loss": 0.4207, | |
| "rewards/chosen": 1.3281025242161106, | |
| "rewards/margins": 2.8707167735120773, | |
| "rewards/rejected": -1.5426142492959665, | |
| "step": 210 | |
| }, | |
| { | |
| "count/fg_chosen": 24.214284896850586, | |
| "count/fg_rejected": 7.5, | |
| "epoch": 0.22564102564102564, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.594059944152832, | |
| "fg_logps/policy_chosen": -6.371434688568115, | |
| "fg_logps/policy_rejected": -7.165874004364014, | |
| "fg_logps/reference_KL": -10.995756149291992, | |
| "fg_logps/reference_chosen": -6.087092399597168, | |
| "fg_logps/reference_rejected": -6.730601787567139, | |
| "fg_loss": 0.7775211334228516, | |
| "fg_rewards/chosen_sum": -0.6547192931175232, | |
| "fg_rewards/rejected_sum": -0.29393261671066284, | |
| "grad_norm": 46.801027582446935, | |
| "kl": 0.0, | |
| "learning_rate": 4.3101482326111746e-07, | |
| "logps/chosen": -344.7091128700658, | |
| "logps/rejected": -398.7763671875, | |
| "loss": 0.4334, | |
| "rewards/chosen": 0.5747735876786081, | |
| "rewards/margins": 2.151842461492782, | |
| "rewards/rejected": -1.577068873814174, | |
| "step": 220 | |
| }, | |
| { | |
| "count/fg_chosen": 29.272727966308594, | |
| "count/fg_rejected": 6.55555534362793, | |
| "epoch": 0.2358974358974359, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -11.965290069580078, | |
| "fg_logps/policy_chosen": -6.299896717071533, | |
| "fg_logps/policy_rejected": -7.80208158493042, | |
| "fg_logps/reference_KL": -10.39778995513916, | |
| "fg_logps/reference_chosen": -5.838742256164551, | |
| "fg_logps/reference_rejected": -7.279135704040527, | |
| "fg_loss": 0.6748415231704712, | |
| "fg_rewards/chosen_sum": -1.2193199396133423, | |
| "fg_rewards/rejected_sum": -0.36254453659057617, | |
| "grad_norm": 45.120354528449354, | |
| "kl": 0.0, | |
| "learning_rate": 4.2531356898517673e-07, | |
| "logps/chosen": -358.53559470663265, | |
| "logps/rejected": -378.2181829637097, | |
| "loss": 0.4775, | |
| "rewards/chosen": 0.5253227389588648, | |
| "rewards/margins": 2.214199475507504, | |
| "rewards/rejected": -1.688876736548639, | |
| "step": 230 | |
| }, | |
| { | |
| "count/fg_chosen": 36.1875, | |
| "count/fg_rejected": 7.199999809265137, | |
| "epoch": 0.24615384615384617, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.929790496826172, | |
| "fg_logps/policy_chosen": -6.825524806976318, | |
| "fg_logps/policy_rejected": -7.702876091003418, | |
| "fg_logps/reference_KL": -11.064438819885254, | |
| "fg_logps/reference_chosen": -6.324741363525391, | |
| "fg_logps/reference_rejected": -7.179370403289795, | |
| "fg_loss": 0.7855690717697144, | |
| "fg_rewards/chosen_sum": -1.7501062154769897, | |
| "fg_rewards/rejected_sum": -0.501429557800293, | |
| "grad_norm": 41.89950021868993, | |
| "kl": 0.26182326674461365, | |
| "learning_rate": 4.1961231470923605e-07, | |
| "logps/chosen": -347.2475725446429, | |
| "logps/rejected": -382.114453125, | |
| "loss": 0.4878, | |
| "rewards/chosen": 1.19210935320173, | |
| "rewards/margins": 1.8935284205845424, | |
| "rewards/rejected": -0.7014190673828125, | |
| "step": 240 | |
| }, | |
| { | |
| "count/fg_chosen": 29.399999618530273, | |
| "count/fg_rejected": 7.266666889190674, | |
| "epoch": 0.2564102564102564, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -14.258040428161621, | |
| "fg_logps/policy_chosen": -6.943975925445557, | |
| "fg_logps/policy_rejected": -7.556437969207764, | |
| "fg_logps/reference_KL": -11.820046424865723, | |
| "fg_logps/reference_chosen": -6.191815376281738, | |
| "fg_logps/reference_rejected": -6.7565178871154785, | |
| "fg_loss": 0.8718132972717285, | |
| "fg_rewards/chosen_sum": -1.6466922760009766, | |
| "fg_rewards/rejected_sum": -0.6005190014839172, | |
| "grad_norm": 41.93986307357718, | |
| "kl": 0.0, | |
| "learning_rate": 4.139110604332953e-07, | |
| "logps/chosen": -327.5843017578125, | |
| "logps/rejected": -385.35126953125, | |
| "loss": 0.4012, | |
| "rewards/chosen": 1.1960113525390625, | |
| "rewards/margins": 2.935526466369629, | |
| "rewards/rejected": -1.7395151138305665, | |
| "step": 250 | |
| }, | |
| { | |
| "count/fg_chosen": 33.38461685180664, | |
| "count/fg_rejected": 7.692307472229004, | |
| "epoch": 0.26666666666666666, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.9034423828125, | |
| "fg_logps/policy_chosen": -6.196410655975342, | |
| "fg_logps/policy_rejected": -7.9534759521484375, | |
| "fg_logps/reference_KL": -10.658266067504883, | |
| "fg_logps/reference_chosen": -5.453469276428223, | |
| "fg_logps/reference_rejected": -7.014803409576416, | |
| "fg_loss": 0.9994122982025146, | |
| "fg_rewards/chosen_sum": -1.9320785999298096, | |
| "fg_rewards/rejected_sum": -0.8277677893638611, | |
| "grad_norm": 32.044513351466335, | |
| "kl": 0.0, | |
| "learning_rate": 4.0820980615735465e-07, | |
| "logps/chosen": -334.1531723484849, | |
| "logps/rejected": -391.7591838430851, | |
| "loss": 0.4248, | |
| "rewards/chosen": 0.9683192859996449, | |
| "rewards/margins": 2.343336492719226, | |
| "rewards/rejected": -1.3750172067195812, | |
| "step": 260 | |
| }, | |
| { | |
| "count/fg_chosen": 25.071428298950195, | |
| "count/fg_rejected": 5.142857074737549, | |
| "epoch": 0.27692307692307694, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -14.50527286529541, | |
| "fg_logps/policy_chosen": -7.0650315284729, | |
| "fg_logps/policy_rejected": -8.667463302612305, | |
| "fg_logps/reference_KL": -12.12157154083252, | |
| "fg_logps/reference_chosen": -6.389377593994141, | |
| "fg_logps/reference_rejected": -7.870638370513916, | |
| "fg_loss": 0.7973106503486633, | |
| "fg_rewards/chosen_sum": -1.2882376909255981, | |
| "fg_rewards/rejected_sum": -0.4027543365955353, | |
| "grad_norm": 68.19722281582398, | |
| "kl": 0.020351696759462357, | |
| "learning_rate": 4.025085518814139e-07, | |
| "logps/chosen": -405.6799411525974, | |
| "logps/rejected": -402.68011106927713, | |
| "loss": 0.4322, | |
| "rewards/chosen": 0.5332601472928926, | |
| "rewards/margins": 2.588090307778585, | |
| "rewards/rejected": -2.0548301604856927, | |
| "step": 270 | |
| }, | |
| { | |
| "count/fg_chosen": 31.959999084472656, | |
| "count/fg_rejected": 9.0, | |
| "epoch": 0.28717948717948716, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.018957138061523, | |
| "fg_logps/policy_chosen": -6.5620503425598145, | |
| "fg_logps/policy_rejected": -8.468265533447266, | |
| "fg_logps/reference_KL": -9.829309463500977, | |
| "fg_logps/reference_chosen": -5.755721569061279, | |
| "fg_logps/reference_rejected": -7.4278693199157715, | |
| "fg_loss": 0.8410596251487732, | |
| "fg_rewards/chosen_sum": -2.1979994773864746, | |
| "fg_rewards/rejected_sum": -1.1406161785125732, | |
| "grad_norm": 42.49380746961441, | |
| "kl": 0.0, | |
| "learning_rate": 3.9680729760547324e-07, | |
| "logps/chosen": -333.30290316358025, | |
| "logps/rejected": -438.11288568037975, | |
| "loss": 0.5378, | |
| "rewards/chosen": 0.4965087513864776, | |
| "rewards/margins": 2.022118021313297, | |
| "rewards/rejected": -1.5256092699268196, | |
| "step": 280 | |
| }, | |
| { | |
| "count/fg_chosen": 31.38888931274414, | |
| "count/fg_rejected": 9.166666984558105, | |
| "epoch": 0.29743589743589743, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.498015403747559, | |
| "fg_logps/policy_chosen": -6.605287551879883, | |
| "fg_logps/policy_rejected": -7.190271377563477, | |
| "fg_logps/reference_KL": -11.353104591369629, | |
| "fg_logps/reference_chosen": -6.0449957847595215, | |
| "fg_logps/reference_rejected": -6.696670055389404, | |
| "fg_loss": 0.9254876971244812, | |
| "fg_rewards/chosen_sum": -1.749000072479248, | |
| "fg_rewards/rejected_sum": -0.5272374153137207, | |
| "grad_norm": 34.10871711923179, | |
| "kl": 0.0, | |
| "learning_rate": 3.9110604332953246e-07, | |
| "logps/chosen": -374.3355087652439, | |
| "logps/rejected": -443.0232371794872, | |
| "loss": 0.4904, | |
| "rewards/chosen": 0.7626230658554449, | |
| "rewards/margins": 2.348516941368766, | |
| "rewards/rejected": -1.5858938755133214, | |
| "step": 290 | |
| }, | |
| { | |
| "count/fg_chosen": 29.75, | |
| "count/fg_rejected": 7.6315789222717285, | |
| "epoch": 0.3076923076923077, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.35108757019043, | |
| "fg_logps/policy_chosen": -6.358473300933838, | |
| "fg_logps/policy_rejected": -7.603394508361816, | |
| "fg_logps/reference_KL": -10.117968559265137, | |
| "fg_logps/reference_chosen": -5.656960964202881, | |
| "fg_logps/reference_rejected": -6.988058090209961, | |
| "fg_loss": 0.687099277973175, | |
| "fg_rewards/chosen_sum": -1.7474342584609985, | |
| "fg_rewards/rejected_sum": -0.6550286412239075, | |
| "grad_norm": 45.87667859445539, | |
| "kl": 0.0, | |
| "learning_rate": 3.854047890535917e-07, | |
| "logps/chosen": -291.94080528846155, | |
| "logps/rejected": -454.1446265243902, | |
| "loss": 0.5046, | |
| "rewards/chosen": -0.05552493608914889, | |
| "rewards/margins": 2.302112236702867, | |
| "rewards/rejected": -2.357637172792016, | |
| "step": 300 | |
| }, | |
| { | |
| "count/fg_chosen": 29.052631378173828, | |
| "count/fg_rejected": 6.157894611358643, | |
| "epoch": 0.31794871794871793, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.878127098083496, | |
| "fg_logps/policy_chosen": -6.794922828674316, | |
| "fg_logps/policy_rejected": -8.655734062194824, | |
| "fg_logps/reference_KL": -11.557811737060547, | |
| "fg_logps/reference_chosen": -6.259448528289795, | |
| "fg_logps/reference_rejected": -7.914809226989746, | |
| "fg_loss": 0.7258095145225525, | |
| "fg_rewards/chosen_sum": -1.6086143255233765, | |
| "fg_rewards/rejected_sum": -0.5489023327827454, | |
| "grad_norm": 43.21903612458155, | |
| "kl": 0.0, | |
| "learning_rate": 3.7970353477765105e-07, | |
| "logps/chosen": -361.2526117369186, | |
| "logps/rejected": -382.5555320945946, | |
| "loss": 0.4464, | |
| "rewards/chosen": 0.8190518756245457, | |
| "rewards/margins": 3.1284712498327383, | |
| "rewards/rejected": -2.3094193742081925, | |
| "step": 310 | |
| }, | |
| { | |
| "count/fg_chosen": 27.66666603088379, | |
| "count/fg_rejected": 7.05555534362793, | |
| "epoch": 0.3282051282051282, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -14.007112503051758, | |
| "fg_logps/policy_chosen": -7.029316425323486, | |
| "fg_logps/policy_rejected": -8.816922187805176, | |
| "fg_logps/reference_KL": -11.414546012878418, | |
| "fg_logps/reference_chosen": -6.1291069984436035, | |
| "fg_logps/reference_rejected": -8.072396278381348, | |
| "fg_loss": 0.6982179880142212, | |
| "fg_rewards/chosen_sum": -1.864067554473877, | |
| "fg_rewards/rejected_sum": -0.625391960144043, | |
| "grad_norm": 49.56835553157457, | |
| "kl": 0.0, | |
| "learning_rate": 3.740022805017103e-07, | |
| "logps/chosen": -308.43726245777026, | |
| "logps/rejected": -462.0808502906977, | |
| "loss": 0.4611, | |
| "rewards/chosen": 0.7936567358068518, | |
| "rewards/margins": 2.709756816430035, | |
| "rewards/rejected": -1.9161000806231832, | |
| "step": 320 | |
| }, | |
| { | |
| "count/fg_chosen": 29.941177368164062, | |
| "count/fg_rejected": 6.125, | |
| "epoch": 0.3384615384615385, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.976774215698242, | |
| "fg_logps/policy_chosen": -6.938152313232422, | |
| "fg_logps/policy_rejected": -7.871105670928955, | |
| "fg_logps/reference_KL": -11.392000198364258, | |
| "fg_logps/reference_chosen": -6.208968162536621, | |
| "fg_logps/reference_rejected": -6.786693096160889, | |
| "fg_loss": 0.8347401022911072, | |
| "fg_rewards/chosen_sum": -2.1311469078063965, | |
| "fg_rewards/rejected_sum": -0.6044603586196899, | |
| "grad_norm": 23.82687521831931, | |
| "kl": 0.0, | |
| "learning_rate": 3.6830102622576964e-07, | |
| "logps/chosen": -316.02463269589555, | |
| "logps/rejected": -366.19430443548384, | |
| "loss": 0.4596, | |
| "rewards/chosen": 0.36891575713655844, | |
| "rewards/margins": 2.952398068754502, | |
| "rewards/rejected": -2.5834823116179435, | |
| "step": 330 | |
| }, | |
| { | |
| "count/fg_chosen": 31.647058486938477, | |
| "count/fg_rejected": 5.0, | |
| "epoch": 0.3487179487179487, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.484383583068848, | |
| "fg_logps/policy_chosen": -6.541139602661133, | |
| "fg_logps/policy_rejected": -8.020905494689941, | |
| "fg_logps/reference_KL": -11.032403945922852, | |
| "fg_logps/reference_chosen": -5.8485236167907715, | |
| "fg_logps/reference_rejected": -7.10052490234375, | |
| "fg_loss": 0.7300294041633606, | |
| "fg_rewards/chosen_sum": -1.9582923650741577, | |
| "fg_rewards/rejected_sum": -0.4550693929195404, | |
| "grad_norm": 44.41360203093714, | |
| "kl": 0.0, | |
| "learning_rate": 3.625997719498289e-07, | |
| "logps/chosen": -337.7124953497024, | |
| "logps/rejected": -364.32930715460526, | |
| "loss": 0.4522, | |
| "rewards/chosen": 0.633344604855492, | |
| "rewards/margins": 2.6144102545907923, | |
| "rewards/rejected": -1.9810656497353, | |
| "step": 340 | |
| }, | |
| { | |
| "count/fg_chosen": 34.35293960571289, | |
| "count/fg_rejected": 8.764705657958984, | |
| "epoch": 0.358974358974359, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -14.230541229248047, | |
| "fg_logps/policy_chosen": -7.154247760772705, | |
| "fg_logps/policy_rejected": -7.5230560302734375, | |
| "fg_logps/reference_KL": -11.367884635925293, | |
| "fg_logps/reference_chosen": -6.23461389541626, | |
| "fg_logps/reference_rejected": -6.495339870452881, | |
| "fg_loss": 0.853543221950531, | |
| "fg_rewards/chosen_sum": -2.904083490371704, | |
| "fg_rewards/rejected_sum": -0.9132155179977417, | |
| "grad_norm": 30.3057952063642, | |
| "kl": 0.0, | |
| "learning_rate": 3.5689851767388824e-07, | |
| "logps/chosen": -394.6584884129214, | |
| "logps/rejected": -393.08568992077466, | |
| "loss": 0.4936, | |
| "rewards/chosen": 0.6523181615250834, | |
| "rewards/margins": 2.7101455334716062, | |
| "rewards/rejected": -2.057827371946523, | |
| "step": 350 | |
| }, | |
| { | |
| "count/fg_chosen": 27.30769157409668, | |
| "count/fg_rejected": 5.0, | |
| "epoch": 0.36923076923076925, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.898691177368164, | |
| "fg_logps/policy_chosen": -7.163309574127197, | |
| "fg_logps/policy_rejected": -6.87624979019165, | |
| "fg_logps/reference_KL": -10.903498649597168, | |
| "fg_logps/reference_chosen": -6.516329288482666, | |
| "fg_logps/reference_rejected": -6.057809829711914, | |
| "fg_loss": 0.7118747234344482, | |
| "fg_rewards/chosen_sum": -1.5648789405822754, | |
| "fg_rewards/rejected_sum": -0.6483681201934814, | |
| "grad_norm": 37.36414008433845, | |
| "kl": 0.0, | |
| "learning_rate": 3.511972633979475e-07, | |
| "logps/chosen": -316.7720240542763, | |
| "logps/rejected": -391.03125, | |
| "loss": 0.4128, | |
| "rewards/chosen": 1.343739258615594, | |
| "rewards/margins": 2.428480033587692, | |
| "rewards/rejected": -1.0847407749720983, | |
| "step": 360 | |
| }, | |
| { | |
| "count/fg_chosen": 31.190475463867188, | |
| "count/fg_rejected": 7.050000190734863, | |
| "epoch": 0.37948717948717947, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.466145515441895, | |
| "fg_logps/policy_chosen": -6.301754951477051, | |
| "fg_logps/policy_rejected": -8.657445907592773, | |
| "fg_logps/reference_KL": -10.610649108886719, | |
| "fg_logps/reference_chosen": -5.920670509338379, | |
| "fg_logps/reference_rejected": -8.133522987365723, | |
| "fg_loss": 0.7434370517730713, | |
| "fg_rewards/chosen_sum": -0.8639131188392639, | |
| "fg_rewards/rejected_sum": -0.45794281363487244, | |
| "grad_norm": 28.069125042652725, | |
| "kl": 0.0, | |
| "learning_rate": 3.4549600912200683e-07, | |
| "logps/chosen": -327.7210542485955, | |
| "logps/rejected": -410.5584286971831, | |
| "loss": 0.4659, | |
| "rewards/chosen": 1.6249963996115695, | |
| "rewards/margins": 2.5865509134622515, | |
| "rewards/rejected": -0.9615545138506822, | |
| "step": 370 | |
| }, | |
| { | |
| "count/fg_chosen": 34.900001525878906, | |
| "count/fg_rejected": 8.899999618530273, | |
| "epoch": 0.38974358974358975, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.721631050109863, | |
| "fg_logps/policy_chosen": -6.37019681930542, | |
| "fg_logps/policy_rejected": -6.883843898773193, | |
| "fg_logps/reference_KL": -11.473298072814941, | |
| "fg_logps/reference_chosen": -6.092167854309082, | |
| "fg_logps/reference_rejected": -6.5457611083984375, | |
| "fg_loss": 0.8677409887313843, | |
| "fg_rewards/chosen_sum": -0.9150064587593079, | |
| "fg_rewards/rejected_sum": -0.41792982816696167, | |
| "grad_norm": 42.9333549511222, | |
| "kl": 0.0, | |
| "learning_rate": 3.397947548460661e-07, | |
| "logps/chosen": -340.5213176448171, | |
| "logps/rejected": -451.7316706730769, | |
| "loss": 0.4514, | |
| "rewards/chosen": 1.505601836413872, | |
| "rewards/margins": 2.4657741472674877, | |
| "rewards/rejected": -0.9601723108536158, | |
| "step": 380 | |
| }, | |
| { | |
| "count/fg_chosen": 29.55555534362793, | |
| "count/fg_rejected": 7.0, | |
| "epoch": 0.4, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.328727722167969, | |
| "fg_logps/policy_chosen": -6.023360729217529, | |
| "fg_logps/policy_rejected": -7.300014972686768, | |
| "fg_logps/reference_KL": -10.421500205993652, | |
| "fg_logps/reference_chosen": -5.618011474609375, | |
| "fg_logps/reference_rejected": -6.620323181152344, | |
| "fg_loss": 0.7781895399093628, | |
| "fg_rewards/chosen_sum": -0.9389697313308716, | |
| "fg_rewards/rejected_sum": -0.5253291130065918, | |
| "grad_norm": 42.84097066535792, | |
| "kl": 0.0, | |
| "learning_rate": 3.340935005701254e-07, | |
| "logps/chosen": -364.2041149400685, | |
| "logps/rejected": -441.3032956178161, | |
| "loss": 0.4992, | |
| "rewards/chosen": 1.0798116187526756, | |
| "rewards/margins": 2.0560204480843476, | |
| "rewards/rejected": -0.9762088293316721, | |
| "step": 390 | |
| }, | |
| { | |
| "count/fg_chosen": 26.72222137451172, | |
| "count/fg_rejected": 6.647058963775635, | |
| "epoch": 0.41025641025641024, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.881956100463867, | |
| "fg_logps/policy_chosen": -6.5600266456604, | |
| "fg_logps/policy_rejected": -9.088929176330566, | |
| "fg_logps/reference_KL": -11.40665054321289, | |
| "fg_logps/reference_chosen": -5.7827043533325195, | |
| "fg_logps/reference_rejected": -8.174890518188477, | |
| "fg_loss": 0.8777969479560852, | |
| "fg_rewards/chosen_sum": -1.556259036064148, | |
| "fg_rewards/rejected_sum": -0.7908374667167664, | |
| "grad_norm": 37.72027450146743, | |
| "kl": 0.0, | |
| "learning_rate": 3.283922462941847e-07, | |
| "logps/chosen": -365.9978794642857, | |
| "logps/rejected": -436.88159722222224, | |
| "loss": 0.4478, | |
| "rewards/chosen": 1.2335292271205358, | |
| "rewards/margins": 2.4691440885029143, | |
| "rewards/rejected": -1.2356148613823785, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.41025641025641024, | |
| "eval_count/fg_chosen": 30.183246612548828, | |
| "eval_count/fg_rejected": 6.92391300201416, | |
| "eval_fg_kl": NaN, | |
| "eval_fg_logps/policy_KL": -13.678318977355957, | |
| "eval_fg_logps/policy_chosen": -6.628693580627441, | |
| "eval_fg_logps/policy_rejected": -8.363188743591309, | |
| "eval_fg_logps/reference_KL": -11.47359848022461, | |
| "eval_fg_logps/reference_chosen": -6.041894912719727, | |
| "eval_fg_logps/reference_rejected": -7.58065938949585, | |
| "eval_fg_loss": 0.7654322385787964, | |
| "eval_fg_rewards/chosen_sum": -1.3938791751861572, | |
| "eval_fg_rewards/rejected_sum": -0.6767725944519043, | |
| "eval_kl": 0.02797871269285679, | |
| "eval_logps/chosen": -340.2313144329897, | |
| "eval_logps/rejected": -400.85385283893396, | |
| "eval_loss": 0.4325231909751892, | |
| "eval_rewards/chosen": 1.316945568665879, | |
| "eval_rewards/margins": 3.0533541780318263, | |
| "eval_rewards/rejected": -1.7364086093659472, | |
| "eval_runtime": 492.9712, | |
| "eval_samples_per_second": 3.515, | |
| "eval_steps_per_second": 0.88, | |
| "step": 400 | |
| }, | |
| { | |
| "count/fg_chosen": 26.549999237060547, | |
| "count/fg_rejected": 6.25, | |
| "epoch": 0.4205128205128205, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -15.237287521362305, | |
| "fg_logps/policy_chosen": -6.820374488830566, | |
| "fg_logps/policy_rejected": -8.927366256713867, | |
| "fg_logps/reference_KL": -12.311280250549316, | |
| "fg_logps/reference_chosen": -5.970030784606934, | |
| "fg_logps/reference_rejected": -7.938845634460449, | |
| "fg_loss": 0.8091492056846619, | |
| "fg_rewards/chosen_sum": -1.5172061920166016, | |
| "fg_rewards/rejected_sum": -0.7380185723304749, | |
| "grad_norm": 53.93549024472509, | |
| "kl": 0.0, | |
| "learning_rate": 3.22690992018244e-07, | |
| "logps/chosen": -324.6869419642857, | |
| "logps/rejected": -405.54951054216866, | |
| "loss": 0.4023, | |
| "rewards/chosen": 1.265897478376116, | |
| "rewards/margins": 3.5306600810328366, | |
| "rewards/rejected": -2.2647626026567207, | |
| "step": 410 | |
| }, | |
| { | |
| "count/fg_chosen": 23.399999618530273, | |
| "count/fg_rejected": 6.133333206176758, | |
| "epoch": 0.4307692307692308, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -14.516840934753418, | |
| "fg_logps/policy_chosen": -7.477798938751221, | |
| "fg_logps/policy_rejected": -8.535691261291504, | |
| "fg_logps/reference_KL": -11.834728240966797, | |
| "fg_logps/reference_chosen": -6.342043876647949, | |
| "fg_logps/reference_rejected": -7.10928201675415, | |
| "fg_loss": 0.8881044387817383, | |
| "fg_rewards/chosen_sum": -2.0730390548706055, | |
| "fg_rewards/rejected_sum": -0.9379479289054871, | |
| "grad_norm": 37.934930263204464, | |
| "kl": 0.04062976688146591, | |
| "learning_rate": 3.169897377423033e-07, | |
| "logps/chosen": -352.2984280873494, | |
| "logps/rejected": -437.3393871753247, | |
| "loss": 0.4353, | |
| "rewards/chosen": 0.722329794642437, | |
| "rewards/margins": 3.3173880871799777, | |
| "rewards/rejected": -2.5950582925375407, | |
| "step": 420 | |
| }, | |
| { | |
| "count/fg_chosen": 29.41176414489746, | |
| "count/fg_rejected": 5.882352828979492, | |
| "epoch": 0.441025641025641, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.61406421661377, | |
| "fg_logps/policy_chosen": -6.908777713775635, | |
| "fg_logps/policy_rejected": -9.259625434875488, | |
| "fg_logps/reference_KL": -10.859848976135254, | |
| "fg_logps/reference_chosen": -5.828268527984619, | |
| "fg_logps/reference_rejected": -7.893514156341553, | |
| "fg_loss": 0.7920488119125366, | |
| "fg_rewards/chosen_sum": -2.7851388454437256, | |
| "fg_rewards/rejected_sum": -0.8430763483047485, | |
| "grad_norm": 31.263236198590103, | |
| "kl": 0.20134501159191132, | |
| "learning_rate": 3.112884834663626e-07, | |
| "logps/chosen": -338.0028831845238, | |
| "logps/rejected": -437.03207236842104, | |
| "loss": 0.4237, | |
| "rewards/chosen": 1.1830097380138578, | |
| "rewards/margins": 3.346872267567723, | |
| "rewards/rejected": -2.163862529553865, | |
| "step": 430 | |
| }, | |
| { | |
| "count/fg_chosen": 31.16666603088379, | |
| "count/fg_rejected": 5.583333492279053, | |
| "epoch": 0.4512820512820513, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.71406078338623, | |
| "fg_logps/policy_chosen": -6.013169765472412, | |
| "fg_logps/policy_rejected": -7.012132167816162, | |
| "fg_logps/reference_KL": -10.454259872436523, | |
| "fg_logps/reference_chosen": -5.288631439208984, | |
| "fg_logps/reference_rejected": -6.411142826080322, | |
| "fg_loss": 0.8265379071235657, | |
| "fg_rewards/chosen_sum": -1.9277740716934204, | |
| "fg_rewards/rejected_sum": -0.4081937372684479, | |
| "grad_norm": 33.498855345311206, | |
| "kl": 0.0, | |
| "learning_rate": 3.055872291904219e-07, | |
| "logps/chosen": -433.9810126582278, | |
| "logps/rejected": -409.45997299382714, | |
| "loss": 0.4124, | |
| "rewards/chosen": 0.23994885215276404, | |
| "rewards/margins": 2.5844254342442956, | |
| "rewards/rejected": -2.3444765820915316, | |
| "step": 440 | |
| }, | |
| { | |
| "count/fg_chosen": 28.214284896850586, | |
| "count/fg_rejected": 7.0714287757873535, | |
| "epoch": 0.46153846153846156, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -14.437283515930176, | |
| "fg_logps/policy_chosen": -7.329289436340332, | |
| "fg_logps/policy_rejected": -7.971861839294434, | |
| "fg_logps/reference_KL": -11.504508018493652, | |
| "fg_logps/reference_chosen": -6.307824611663818, | |
| "fg_logps/reference_rejected": -7.138981342315674, | |
| "fg_loss": 0.8858200907707214, | |
| "fg_rewards/chosen_sum": -2.4875144958496094, | |
| "fg_rewards/rejected_sum": -0.8280299305915833, | |
| "grad_norm": 31.189671182424355, | |
| "kl": 0.0, | |
| "learning_rate": 2.998859749144812e-07, | |
| "logps/chosen": -298.6210195806962, | |
| "logps/rejected": -419.6058545524691, | |
| "loss": 0.4201, | |
| "rewards/chosen": 0.8161652963372725, | |
| "rewards/margins": 3.091305375788468, | |
| "rewards/rejected": -2.275140079451196, | |
| "step": 450 | |
| }, | |
| { | |
| "count/fg_chosen": 32.3636360168457, | |
| "count/fg_rejected": 4.7272725105285645, | |
| "epoch": 0.4717948717948718, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.52069091796875, | |
| "fg_logps/policy_chosen": -6.610226154327393, | |
| "fg_logps/policy_rejected": -10.518632888793945, | |
| "fg_logps/reference_KL": -10.879704475402832, | |
| "fg_logps/reference_chosen": -5.88496732711792, | |
| "fg_logps/reference_rejected": -9.407367706298828, | |
| "fg_loss": 0.6551663279533386, | |
| "fg_rewards/chosen_sum": -2.258774757385254, | |
| "fg_rewards/rejected_sum": -0.5948446989059448, | |
| "grad_norm": 33.51878345246348, | |
| "kl": 0.029797697439789772, | |
| "learning_rate": 2.941847206385404e-07, | |
| "logps/chosen": -332.60402610085225, | |
| "logps/rejected": -395.40771484375, | |
| "loss": 0.4001, | |
| "rewards/chosen": 1.4196222478693181, | |
| "rewards/margins": 3.0095812864977907, | |
| "rewards/rejected": -1.5899590386284723, | |
| "step": 460 | |
| }, | |
| { | |
| "count/fg_chosen": 31.5, | |
| "count/fg_rejected": 5.800000190734863, | |
| "epoch": 0.48205128205128206, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.285706520080566, | |
| "fg_logps/policy_chosen": -6.048055648803711, | |
| "fg_logps/policy_rejected": -8.306843757629395, | |
| "fg_logps/reference_KL": -10.14644718170166, | |
| "fg_logps/reference_chosen": -5.535238742828369, | |
| "fg_logps/reference_rejected": -7.302800178527832, | |
| "fg_loss": 0.6428090333938599, | |
| "fg_rewards/chosen_sum": -1.36484956741333, | |
| "fg_rewards/rejected_sum": -0.6093672513961792, | |
| "grad_norm": 26.805133801472735, | |
| "kl": 0.17673882842063904, | |
| "learning_rate": 2.8848346636259974e-07, | |
| "logps/chosen": -317.91790291432585, | |
| "logps/rejected": -368.47114326584506, | |
| "loss": 0.4744, | |
| "rewards/chosen": 1.5331906093640273, | |
| "rewards/margins": 2.4303819928332997, | |
| "rewards/rejected": -0.8971913834692726, | |
| "step": 470 | |
| }, | |
| { | |
| "count/fg_chosen": 39.266666412353516, | |
| "count/fg_rejected": 7.4666666984558105, | |
| "epoch": 0.49230769230769234, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.738459587097168, | |
| "fg_logps/policy_chosen": -6.287077903747559, | |
| "fg_logps/policy_rejected": -6.787537097930908, | |
| "fg_logps/reference_KL": -11.216691017150879, | |
| "fg_logps/reference_chosen": -5.826966762542725, | |
| "fg_logps/reference_rejected": -6.379599571228027, | |
| "fg_loss": 0.7447641491889954, | |
| "fg_rewards/chosen_sum": -0.980557382106781, | |
| "fg_rewards/rejected_sum": -0.506367564201355, | |
| "grad_norm": 39.97199247238054, | |
| "kl": 0.0, | |
| "learning_rate": 2.82782212086659e-07, | |
| "logps/chosen": -401.7761665239726, | |
| "logps/rejected": -363.2634698275862, | |
| "loss": 0.5103, | |
| "rewards/chosen": 1.34750000418049, | |
| "rewards/margins": 1.8919780588848274, | |
| "rewards/rejected": -0.5444780547043373, | |
| "step": 480 | |
| }, | |
| { | |
| "count/fg_chosen": 26.875, | |
| "count/fg_rejected": 5.400000095367432, | |
| "epoch": 0.5025641025641026, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -16.641780853271484, | |
| "fg_logps/policy_chosen": -8.056283950805664, | |
| "fg_logps/policy_rejected": -8.251357078552246, | |
| "fg_logps/reference_KL": -13.59717845916748, | |
| "fg_logps/reference_chosen": -7.162622928619385, | |
| "fg_logps/reference_rejected": -7.323818683624268, | |
| "fg_loss": 0.8411279916763306, | |
| "fg_rewards/chosen_sum": -1.9043647050857544, | |
| "fg_rewards/rejected_sum": -0.7839928269386292, | |
| "grad_norm": 29.60091559742249, | |
| "kl": 0.22372007369995117, | |
| "learning_rate": 2.7708095781071834e-07, | |
| "logps/chosen": -324.56468441611844, | |
| "logps/rejected": -461.18638392857144, | |
| "loss": 0.4437, | |
| "rewards/chosen": 1.3998164126747532, | |
| "rewards/margins": 2.890947968141178, | |
| "rewards/rejected": -1.4911315554664248, | |
| "step": 490 | |
| }, | |
| { | |
| "count/fg_chosen": 27.3157901763916, | |
| "count/fg_rejected": 5.294117450714111, | |
| "epoch": 0.5128205128205128, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -14.019055366516113, | |
| "fg_logps/policy_chosen": -6.862349033355713, | |
| "fg_logps/policy_rejected": -8.392266273498535, | |
| "fg_logps/reference_KL": -11.065834999084473, | |
| "fg_logps/reference_chosen": -6.182069301605225, | |
| "fg_logps/reference_rejected": -7.583798408508301, | |
| "fg_loss": 0.8486608266830444, | |
| "fg_rewards/chosen_sum": -1.7188913822174072, | |
| "fg_rewards/rejected_sum": -0.4710962176322937, | |
| "grad_norm": 37.93308715806046, | |
| "kl": 0.0, | |
| "learning_rate": 2.713797035347776e-07, | |
| "logps/chosen": -336.33485504518075, | |
| "logps/rejected": -412.9320211038961, | |
| "loss": 0.4063, | |
| "rewards/chosen": 1.6464567988751881, | |
| "rewards/margins": 3.429747315121637, | |
| "rewards/rejected": -1.783290516246449, | |
| "step": 500 | |
| }, | |
| { | |
| "count/fg_chosen": 32.52941131591797, | |
| "count/fg_rejected": 6.1875, | |
| "epoch": 0.5230769230769231, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -14.041118621826172, | |
| "fg_logps/policy_chosen": -6.475778579711914, | |
| "fg_logps/policy_rejected": -8.933878898620605, | |
| "fg_logps/reference_KL": -11.386185646057129, | |
| "fg_logps/reference_chosen": -6.088446617126465, | |
| "fg_logps/reference_rejected": -8.2723970413208, | |
| "fg_loss": 0.6639065742492676, | |
| "fg_rewards/chosen_sum": -1.039247989654541, | |
| "fg_rewards/rejected_sum": -0.31398898363113403, | |
| "grad_norm": 51.003351409032156, | |
| "kl": 0.0, | |
| "learning_rate": 2.6567844925883693e-07, | |
| "logps/chosen": -315.5387290396341, | |
| "logps/rejected": -434.9411057692308, | |
| "loss": 0.4328, | |
| "rewards/chosen": 1.3493434626881669, | |
| "rewards/margins": 2.8553199195503964, | |
| "rewards/rejected": -1.5059764568622296, | |
| "step": 510 | |
| }, | |
| { | |
| "count/fg_chosen": 29.549999237060547, | |
| "count/fg_rejected": 7.300000190734863, | |
| "epoch": 0.5333333333333333, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -15.461477279663086, | |
| "fg_logps/policy_chosen": -7.2638678550720215, | |
| "fg_logps/policy_rejected": -8.44337272644043, | |
| "fg_logps/reference_KL": -12.010942459106445, | |
| "fg_logps/reference_chosen": -6.417178153991699, | |
| "fg_logps/reference_rejected": -7.042668342590332, | |
| "fg_loss": 0.8494647145271301, | |
| "fg_rewards/chosen_sum": -2.0671496391296387, | |
| "fg_rewards/rejected_sum": -0.9835360646247864, | |
| "grad_norm": 41.33647069198984, | |
| "kl": 0.0, | |
| "learning_rate": 2.599771949828962e-07, | |
| "logps/chosen": -332.418183117378, | |
| "logps/rejected": -371.52498998397436, | |
| "loss": 0.455, | |
| "rewards/chosen": 1.320475787651248, | |
| "rewards/margins": 3.076387394659962, | |
| "rewards/rejected": -1.755911607008714, | |
| "step": 520 | |
| }, | |
| { | |
| "count/fg_chosen": 30.526315689086914, | |
| "count/fg_rejected": 9.11111068725586, | |
| "epoch": 0.5435897435897435, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.753694534301758, | |
| "fg_logps/policy_chosen": -6.313483238220215, | |
| "fg_logps/policy_rejected": -8.319221496582031, | |
| "fg_logps/reference_KL": -10.8753080368042, | |
| "fg_logps/reference_chosen": -5.6761603355407715, | |
| "fg_logps/reference_rejected": -7.430839538574219, | |
| "fg_loss": 0.7786957621574402, | |
| "fg_rewards/chosen_sum": -1.3800252676010132, | |
| "fg_rewards/rejected_sum": -1.1956380605697632, | |
| "grad_norm": 34.93340197656234, | |
| "kl": 0.0, | |
| "learning_rate": 2.542759407069555e-07, | |
| "logps/chosen": -311.92927758487656, | |
| "logps/rejected": -372.22604331487344, | |
| "loss": 0.448, | |
| "rewards/chosen": 1.0696545824592496, | |
| "rewards/margins": 3.1661021045864253, | |
| "rewards/rejected": -2.0964475221271757, | |
| "step": 530 | |
| }, | |
| { | |
| "count/fg_chosen": 26.764705657958984, | |
| "count/fg_rejected": 6.352941036224365, | |
| "epoch": 0.5538461538461539, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.03756046295166, | |
| "fg_logps/policy_chosen": -6.464415073394775, | |
| "fg_logps/policy_rejected": -7.652871608734131, | |
| "fg_logps/reference_KL": -10.35094165802002, | |
| "fg_logps/reference_chosen": -5.7773261070251465, | |
| "fg_logps/reference_rejected": -6.825320720672607, | |
| "fg_loss": 0.8047051429748535, | |
| "fg_rewards/chosen_sum": -1.6725194454193115, | |
| "fg_rewards/rejected_sum": -0.7678513526916504, | |
| "grad_norm": 41.240539642737886, | |
| "kl": 0.0, | |
| "learning_rate": 2.485746864310148e-07, | |
| "logps/chosen": -345.1589215158046, | |
| "logps/rejected": -397.16462435787673, | |
| "loss": 0.4683, | |
| "rewards/chosen": 1.1091806696749282, | |
| "rewards/margins": 2.9815141440301294, | |
| "rewards/rejected": -1.8723334743552011, | |
| "step": 540 | |
| }, | |
| { | |
| "count/fg_chosen": 36.0, | |
| "count/fg_rejected": 7.176470756530762, | |
| "epoch": 0.5641025641025641, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.34708023071289, | |
| "fg_logps/policy_chosen": -6.144561767578125, | |
| "fg_logps/policy_rejected": -7.657267093658447, | |
| "fg_logps/reference_KL": -10.591930389404297, | |
| "fg_logps/reference_chosen": -5.485869884490967, | |
| "fg_logps/reference_rejected": -6.727408409118652, | |
| "fg_loss": 0.7655860185623169, | |
| "fg_rewards/chosen_sum": -1.8279484510421753, | |
| "fg_rewards/rejected_sum": -0.5319306254386902, | |
| "grad_norm": 34.418931407344175, | |
| "kl": 0.0, | |
| "learning_rate": 2.428734321550741e-07, | |
| "logps/chosen": -332.24548669763516, | |
| "logps/rejected": -384.9080214389535, | |
| "loss": 0.4533, | |
| "rewards/chosen": 1.4185297166978992, | |
| "rewards/margins": 3.59055198175513, | |
| "rewards/rejected": -2.172022265057231, | |
| "step": 550 | |
| }, | |
| { | |
| "count/fg_chosen": 30.6842098236084, | |
| "count/fg_rejected": 5.842105388641357, | |
| "epoch": 0.5743589743589743, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.044573783874512, | |
| "fg_logps/policy_chosen": -6.35300874710083, | |
| "fg_logps/policy_rejected": -8.149062156677246, | |
| "fg_logps/reference_KL": -9.564573287963867, | |
| "fg_logps/reference_chosen": -5.577574253082275, | |
| "fg_logps/reference_rejected": -7.246463298797607, | |
| "fg_loss": 0.7160053849220276, | |
| "fg_rewards/chosen_sum": -1.8285123109817505, | |
| "fg_rewards/rejected_sum": -0.428337961435318, | |
| "grad_norm": 32.969063143022574, | |
| "kl": 0.0, | |
| "learning_rate": 2.371721778791334e-07, | |
| "logps/chosen": -404.944683908046, | |
| "logps/rejected": -394.23758561643837, | |
| "loss": 0.4521, | |
| "rewards/chosen": 1.1878378857141254, | |
| "rewards/margins": 2.883744642389532, | |
| "rewards/rejected": -1.6959067566754067, | |
| "step": 560 | |
| }, | |
| { | |
| "count/fg_chosen": 29.5, | |
| "count/fg_rejected": 6.733333110809326, | |
| "epoch": 0.5846153846153846, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.527681350708008, | |
| "fg_logps/policy_chosen": -6.548896789550781, | |
| "fg_logps/policy_rejected": -6.040011882781982, | |
| "fg_logps/reference_KL": -9.865092277526855, | |
| "fg_logps/reference_chosen": -5.893582344055176, | |
| "fg_logps/reference_rejected": -5.497416019439697, | |
| "fg_loss": 0.5755335092544556, | |
| "fg_rewards/chosen_sum": -1.5131157636642456, | |
| "fg_rewards/rejected_sum": -0.35849064588546753, | |
| "grad_norm": 40.43474566516004, | |
| "kl": 0.0, | |
| "learning_rate": 2.314709236031927e-07, | |
| "logps/chosen": -354.1348353794643, | |
| "logps/rejected": -417.60911800986844, | |
| "loss": 0.4105, | |
| "rewards/chosen": 0.7818209330240885, | |
| "rewards/margins": 3.425596471418414, | |
| "rewards/rejected": -2.6437755383943258, | |
| "step": 570 | |
| }, | |
| { | |
| "count/fg_chosen": 31.428571701049805, | |
| "count/fg_rejected": 7.599999904632568, | |
| "epoch": 0.5948717948717949, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.412928581237793, | |
| "fg_logps/policy_chosen": -6.371100902557373, | |
| "fg_logps/policy_rejected": -7.6736016273498535, | |
| "fg_logps/reference_KL": -10.896791458129883, | |
| "fg_logps/reference_chosen": -5.65976095199585, | |
| "fg_logps/reference_rejected": -6.982507228851318, | |
| "fg_loss": 0.7268858551979065, | |
| "fg_rewards/chosen_sum": -1.6253752708435059, | |
| "fg_rewards/rejected_sum": -0.8460947871208191, | |
| "grad_norm": 57.39186053322085, | |
| "kl": 0.0, | |
| "learning_rate": 2.2576966932725198e-07, | |
| "logps/chosen": -291.66895736882714, | |
| "logps/rejected": -384.9341623813291, | |
| "loss": 0.4683, | |
| "rewards/chosen": 1.6188120900848766, | |
| "rewards/margins": 2.9102534159847977, | |
| "rewards/rejected": -1.2914413258999209, | |
| "step": 580 | |
| }, | |
| { | |
| "count/fg_chosen": 28.904762268066406, | |
| "count/fg_rejected": 6.526315689086914, | |
| "epoch": 0.6051282051282051, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.970428466796875, | |
| "fg_logps/policy_chosen": -6.568854808807373, | |
| "fg_logps/policy_rejected": -8.221002578735352, | |
| "fg_logps/reference_KL": -11.08731746673584, | |
| "fg_logps/reference_chosen": -5.649308204650879, | |
| "fg_logps/reference_rejected": -7.402557373046875, | |
| "fg_loss": 0.705423891544342, | |
| "fg_rewards/chosen_sum": -2.060758590698242, | |
| "fg_rewards/rejected_sum": -0.8252547979354858, | |
| "grad_norm": 35.44361919546434, | |
| "kl": 0.0, | |
| "learning_rate": 2.2006841505131128e-07, | |
| "logps/chosen": -442.12862723214283, | |
| "logps/rejected": -406.4741981907895, | |
| "loss": 0.4365, | |
| "rewards/chosen": 1.5486488342285156, | |
| "rewards/margins": 3.623551418906764, | |
| "rewards/rejected": -2.0749025846782483, | |
| "step": 590 | |
| }, | |
| { | |
| "count/fg_chosen": 27.83333396911621, | |
| "count/fg_rejected": 6.2727274894714355, | |
| "epoch": 0.6153846153846154, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -14.578009605407715, | |
| "fg_logps/policy_chosen": -7.306002140045166, | |
| "fg_logps/policy_rejected": -8.329404830932617, | |
| "fg_logps/reference_KL": -11.412123680114746, | |
| "fg_logps/reference_chosen": -6.204747676849365, | |
| "fg_logps/reference_rejected": -6.661261081695557, | |
| "fg_loss": 0.649915337562561, | |
| "fg_rewards/chosen_sum": -2.443979263305664, | |
| "fg_rewards/rejected_sum": -0.9118065237998962, | |
| "grad_norm": 48.33661978525442, | |
| "kl": 0.0, | |
| "learning_rate": 2.1436716077537057e-07, | |
| "logps/chosen": -353.3546720805921, | |
| "logps/rejected": -475.7469773065476, | |
| "loss": 0.4239, | |
| "rewards/chosen": 1.3294219970703125, | |
| "rewards/margins": 3.233419145856585, | |
| "rewards/rejected": -1.9039971487862724, | |
| "step": 600 | |
| }, | |
| { | |
| "count/fg_chosen": 25.647058486938477, | |
| "count/fg_rejected": 6.470588207244873, | |
| "epoch": 0.6256410256410256, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.512959480285645, | |
| "fg_logps/policy_chosen": -6.967007160186768, | |
| "fg_logps/policy_rejected": -8.427515029907227, | |
| "fg_logps/reference_KL": -10.780054092407227, | |
| "fg_logps/reference_chosen": -6.145755290985107, | |
| "fg_logps/reference_rejected": -7.377350807189941, | |
| "fg_loss": 0.8846892714500427, | |
| "fg_rewards/chosen_sum": -1.872863531112671, | |
| "fg_rewards/rejected_sum": -0.8110222816467285, | |
| "grad_norm": 27.94628622877484, | |
| "kl": 0.0, | |
| "learning_rate": 2.0866590649942987e-07, | |
| "logps/chosen": -325.96284054487177, | |
| "logps/rejected": -358.4112280868902, | |
| "loss": 0.443, | |
| "rewards/chosen": 1.2946516183706431, | |
| "rewards/margins": 2.8088877894417656, | |
| "rewards/rejected": -1.5142361710711223, | |
| "step": 610 | |
| }, | |
| { | |
| "count/fg_chosen": 34.238094329833984, | |
| "count/fg_rejected": 6.949999809265137, | |
| "epoch": 0.6358974358974359, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -14.278539657592773, | |
| "fg_logps/policy_chosen": -6.847230434417725, | |
| "fg_logps/policy_rejected": -9.253725051879883, | |
| "fg_logps/reference_KL": -11.118898391723633, | |
| "fg_logps/reference_chosen": -5.77498722076416, | |
| "fg_logps/reference_rejected": -7.873915195465088, | |
| "fg_loss": 0.7475059628486633, | |
| "fg_rewards/chosen_sum": -2.7328929901123047, | |
| "fg_rewards/rejected_sum": -1.1806801557540894, | |
| "grad_norm": 37.21517260596127, | |
| "kl": 0.0, | |
| "learning_rate": 2.0296465222348917e-07, | |
| "logps/chosen": -345.37012924382714, | |
| "logps/rejected": -452.38132911392404, | |
| "loss": 0.4182, | |
| "rewards/chosen": 2.110634132667824, | |
| "rewards/margins": 4.059031805296879, | |
| "rewards/rejected": -1.9483976726290546, | |
| "step": 620 | |
| }, | |
| { | |
| "count/fg_chosen": 28.3125, | |
| "count/fg_rejected": 7.4375, | |
| "epoch": 0.6461538461538462, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -15.633122444152832, | |
| "fg_logps/policy_chosen": -6.758295059204102, | |
| "fg_logps/policy_rejected": -8.180941581726074, | |
| "fg_logps/reference_KL": -12.073482513427734, | |
| "fg_logps/reference_chosen": -5.663504600524902, | |
| "fg_logps/reference_rejected": -6.7309250831604, | |
| "fg_loss": 0.8814060091972351, | |
| "fg_rewards/chosen_sum": -2.544795513153076, | |
| "fg_rewards/rejected_sum": -1.284183144569397, | |
| "grad_norm": 42.32817075648944, | |
| "kl": 0.0, | |
| "learning_rate": 1.9726339794754846e-07, | |
| "logps/chosen": -351.775993441358, | |
| "logps/rejected": -522.9557950949367, | |
| "loss": 0.4343, | |
| "rewards/chosen": 1.261369964222849, | |
| "rewards/margins": 3.565065836083015, | |
| "rewards/rejected": -2.303695871860166, | |
| "step": 630 | |
| }, | |
| { | |
| "count/fg_chosen": 33.64706039428711, | |
| "count/fg_rejected": 8.058823585510254, | |
| "epoch": 0.6564102564102564, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.795089721679688, | |
| "fg_logps/policy_chosen": -6.418089389801025, | |
| "fg_logps/policy_rejected": -8.086543083190918, | |
| "fg_logps/reference_KL": -10.756563186645508, | |
| "fg_logps/reference_chosen": -5.482754230499268, | |
| "fg_logps/reference_rejected": -6.624981880187988, | |
| "fg_loss": 0.8172480463981628, | |
| "fg_rewards/chosen_sum": -2.2949178218841553, | |
| "fg_rewards/rejected_sum": -1.4898707866668701, | |
| "grad_norm": 49.06648139657291, | |
| "kl": 0.0, | |
| "learning_rate": 1.9156214367160776e-07, | |
| "logps/chosen": -323.18095703125, | |
| "logps/rejected": -457.679443359375, | |
| "loss": 0.3975, | |
| "rewards/chosen": 1.7368663787841796, | |
| "rewards/margins": 4.7420207977294915, | |
| "rewards/rejected": -3.0051544189453123, | |
| "step": 640 | |
| }, | |
| { | |
| "count/fg_chosen": 26.785715103149414, | |
| "count/fg_rejected": 6.0, | |
| "epoch": 0.6666666666666666, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -16.923561096191406, | |
| "fg_logps/policy_chosen": -7.209476470947266, | |
| "fg_logps/policy_rejected": -8.816498756408691, | |
| "fg_logps/reference_KL": -12.868348121643066, | |
| "fg_logps/reference_chosen": -5.93485164642334, | |
| "fg_logps/reference_rejected": -7.260035991668701, | |
| "fg_loss": 0.7890381217002869, | |
| "fg_rewards/chosen_sum": -2.429896831512451, | |
| "fg_rewards/rejected_sum": -0.9942983388900757, | |
| "grad_norm": 42.684857833622196, | |
| "kl": 0.0, | |
| "learning_rate": 1.8586088939566706e-07, | |
| "logps/chosen": -307.12525576636904, | |
| "logps/rejected": -396.83095189144734, | |
| "loss": 0.4325, | |
| "rewards/chosen": 1.7140017918178014, | |
| "rewards/margins": 3.020266568750367, | |
| "rewards/rejected": -1.3062647769325657, | |
| "step": 650 | |
| }, | |
| { | |
| "count/fg_chosen": 32.578948974609375, | |
| "count/fg_rejected": 5.5789475440979, | |
| "epoch": 0.676923076923077, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.829089164733887, | |
| "fg_logps/policy_chosen": -6.605353832244873, | |
| "fg_logps/policy_rejected": -8.679577827453613, | |
| "fg_logps/reference_KL": -11.020931243896484, | |
| "fg_logps/reference_chosen": -5.843183517456055, | |
| "fg_logps/reference_rejected": -7.641061305999756, | |
| "fg_loss": 0.7101105451583862, | |
| "fg_rewards/chosen_sum": -1.6802619695663452, | |
| "fg_rewards/rejected_sum": -0.5640377998352051, | |
| "grad_norm": 51.20082166149615, | |
| "kl": 0.0, | |
| "learning_rate": 1.8015963511972635e-07, | |
| "logps/chosen": -326.6531840479651, | |
| "logps/rejected": -467.38508234797297, | |
| "loss": 0.3955, | |
| "rewards/chosen": 1.7110205362009447, | |
| "rewards/margins": 3.708857912151114, | |
| "rewards/rejected": -1.9978373759501689, | |
| "step": 660 | |
| }, | |
| { | |
| "count/fg_chosen": 34.52941131591797, | |
| "count/fg_rejected": 8.882352828979492, | |
| "epoch": 0.6871794871794872, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -15.263018608093262, | |
| "fg_logps/policy_chosen": -7.245337009429932, | |
| "fg_logps/policy_rejected": -10.788996696472168, | |
| "fg_logps/reference_KL": -11.636683464050293, | |
| "fg_logps/reference_chosen": -6.2825164794921875, | |
| "fg_logps/reference_rejected": -8.302498817443848, | |
| "fg_loss": 0.9282689094543457, | |
| "fg_rewards/chosen_sum": -2.5015087127685547, | |
| "fg_rewards/rejected_sum": -1.4177420139312744, | |
| "grad_norm": 43.77061774737123, | |
| "kl": 0.16583053767681122, | |
| "learning_rate": 1.7445838084378562e-07, | |
| "logps/chosen": -347.34482020547944, | |
| "logps/rejected": -440.41316451149424, | |
| "loss": 0.4428, | |
| "rewards/chosen": 1.4226540343402183, | |
| "rewards/margins": 3.992055567094349, | |
| "rewards/rejected": -2.5694015327541306, | |
| "step": 670 | |
| }, | |
| { | |
| "count/fg_chosen": 33.0625, | |
| "count/fg_rejected": 6.1875, | |
| "epoch": 0.6974358974358974, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.69076156616211, | |
| "fg_logps/policy_chosen": -6.070934295654297, | |
| "fg_logps/policy_rejected": -9.86292552947998, | |
| "fg_logps/reference_KL": -10.596210479736328, | |
| "fg_logps/reference_chosen": -5.411348342895508, | |
| "fg_logps/reference_rejected": -8.194981575012207, | |
| "fg_loss": 0.7803856730461121, | |
| "fg_rewards/chosen_sum": -1.5216065645217896, | |
| "fg_rewards/rejected_sum": -0.7260585427284241, | |
| "grad_norm": 41.07172408412241, | |
| "kl": 0.0, | |
| "learning_rate": 1.6875712656784492e-07, | |
| "logps/chosen": -321.7051943824405, | |
| "logps/rejected": -402.75840357730266, | |
| "loss": 0.431, | |
| "rewards/chosen": 1.6818878537132627, | |
| "rewards/margins": 2.9571292035860526, | |
| "rewards/rejected": -1.27524134987279, | |
| "step": 680 | |
| }, | |
| { | |
| "count/fg_chosen": 33.04166793823242, | |
| "count/fg_rejected": 7.956521511077881, | |
| "epoch": 0.7076923076923077, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.681419372558594, | |
| "fg_logps/policy_chosen": -6.311405181884766, | |
| "fg_logps/policy_rejected": -8.458182334899902, | |
| "fg_logps/reference_KL": -10.43921947479248, | |
| "fg_logps/reference_chosen": -5.538773059844971, | |
| "fg_logps/reference_rejected": -7.221550941467285, | |
| "fg_loss": 0.774190366268158, | |
| "fg_rewards/chosen_sum": -2.0744409561157227, | |
| "fg_rewards/rejected_sum": -1.0601459741592407, | |
| "grad_norm": 26.76398723713879, | |
| "kl": 0.0, | |
| "learning_rate": 1.6305587229190422e-07, | |
| "logps/chosen": -337.47130408653845, | |
| "logps/rejected": -426.1006573932927, | |
| "loss": 0.4689, | |
| "rewards/chosen": 1.7125216753054888, | |
| "rewards/margins": 3.957819735280718, | |
| "rewards/rejected": -2.245298059975229, | |
| "step": 690 | |
| }, | |
| { | |
| "count/fg_chosen": 29.933332443237305, | |
| "count/fg_rejected": 7.5, | |
| "epoch": 0.717948717948718, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -18.62053871154785, | |
| "fg_logps/policy_chosen": -6.723959445953369, | |
| "fg_logps/policy_rejected": -8.809980392456055, | |
| "fg_logps/reference_KL": -14.621339797973633, | |
| "fg_logps/reference_chosen": -5.710059642791748, | |
| "fg_logps/reference_rejected": -7.013789176940918, | |
| "fg_loss": 0.7358676791191101, | |
| "fg_rewards/chosen_sum": -2.391604423522949, | |
| "fg_rewards/rejected_sum": -1.1176395416259766, | |
| "grad_norm": 32.71207561201088, | |
| "kl": 0.0, | |
| "learning_rate": 1.573546180159635e-07, | |
| "logps/chosen": -377.94707661290323, | |
| "logps/rejected": -423.6705923507463, | |
| "loss": 0.3983, | |
| "rewards/chosen": 1.9157637729439685, | |
| "rewards/margins": 3.7593996736406803, | |
| "rewards/rejected": -1.8436359006967118, | |
| "step": 700 | |
| }, | |
| { | |
| "count/fg_chosen": 34.0, | |
| "count/fg_rejected": 7.0, | |
| "epoch": 0.7282051282051282, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.678239822387695, | |
| "fg_logps/policy_chosen": -5.730792999267578, | |
| "fg_logps/policy_rejected": -6.297366142272949, | |
| "fg_logps/reference_KL": -10.122818946838379, | |
| "fg_logps/reference_chosen": -5.305339813232422, | |
| "fg_logps/reference_rejected": -5.903895378112793, | |
| "fg_loss": 0.8335784673690796, | |
| "fg_rewards/chosen_sum": -1.278421401977539, | |
| "fg_rewards/rejected_sum": -0.4984094202518463, | |
| "grad_norm": 30.093125040941494, | |
| "kl": 0.0, | |
| "learning_rate": 1.516533637400228e-07, | |
| "logps/chosen": -311.1296672077922, | |
| "logps/rejected": -378.9050263554217, | |
| "loss": 0.4411, | |
| "rewards/chosen": 1.7025673606178977, | |
| "rewards/margins": 4.042804529998545, | |
| "rewards/rejected": -2.3402371693806474, | |
| "step": 710 | |
| }, | |
| { | |
| "count/fg_chosen": 25.214284896850586, | |
| "count/fg_rejected": 4.538461685180664, | |
| "epoch": 0.7384615384615385, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -15.262101173400879, | |
| "fg_logps/policy_chosen": -6.9626898765563965, | |
| "fg_logps/policy_rejected": -9.714632034301758, | |
| "fg_logps/reference_KL": -11.78027057647705, | |
| "fg_logps/reference_chosen": -6.246325969696045, | |
| "fg_logps/reference_rejected": -8.87566089630127, | |
| "fg_loss": 0.7088484764099121, | |
| "fg_rewards/chosen_sum": -1.5614358186721802, | |
| "fg_rewards/rejected_sum": -0.42503052949905396, | |
| "grad_norm": 41.939062686169684, | |
| "kl": 0.0, | |
| "learning_rate": 1.459521094640821e-07, | |
| "logps/chosen": -357.324462890625, | |
| "logps/rejected": -446.333984375, | |
| "loss": 0.3344, | |
| "rewards/chosen": 1.5038203239440917, | |
| "rewards/margins": 4.055677318572998, | |
| "rewards/rejected": -2.5518569946289062, | |
| "step": 720 | |
| }, | |
| { | |
| "count/fg_chosen": 34.75, | |
| "count/fg_rejected": 8.470588684082031, | |
| "epoch": 0.7487179487179487, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.44446086883545, | |
| "fg_logps/policy_chosen": -6.577872276306152, | |
| "fg_logps/policy_rejected": -8.81990909576416, | |
| "fg_logps/reference_KL": -10.716168403625488, | |
| "fg_logps/reference_chosen": -6.01681661605835, | |
| "fg_logps/reference_rejected": -8.056208610534668, | |
| "fg_loss": 0.7597875595092773, | |
| "fg_rewards/chosen_sum": -1.6217119693756104, | |
| "fg_rewards/rejected_sum": -1.0568969249725342, | |
| "grad_norm": 29.217422708191275, | |
| "kl": 0.0, | |
| "learning_rate": 1.402508551881414e-07, | |
| "logps/chosen": -325.07026041666666, | |
| "logps/rejected": -434.04232536764704, | |
| "loss": 0.3915, | |
| "rewards/chosen": 2.13581298828125, | |
| "rewards/margins": 5.36644473805147, | |
| "rewards/rejected": -3.2306317497702204, | |
| "step": 730 | |
| }, | |
| { | |
| "count/fg_chosen": 32.095237731933594, | |
| "count/fg_rejected": 8.7619047164917, | |
| "epoch": 0.7589743589743589, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -16.643653869628906, | |
| "fg_logps/policy_chosen": -7.633936405181885, | |
| "fg_logps/policy_rejected": -8.764749526977539, | |
| "fg_logps/reference_KL": -12.597454071044922, | |
| "fg_logps/reference_chosen": -6.6152801513671875, | |
| "fg_logps/reference_rejected": -7.493948936462402, | |
| "fg_loss": 0.8724325895309448, | |
| "fg_rewards/chosen_sum": -2.190063714981079, | |
| "fg_rewards/rejected_sum": -1.0442728996276855, | |
| "grad_norm": 35.508557979266605, | |
| "kl": 0.0, | |
| "learning_rate": 1.345496009122007e-07, | |
| "logps/chosen": -323.4399604301948, | |
| "logps/rejected": -447.21136106927713, | |
| "loss": 0.4821, | |
| "rewards/chosen": 1.2747364787312296, | |
| "rewards/margins": 3.8379994181843546, | |
| "rewards/rejected": -2.563262939453125, | |
| "step": 740 | |
| }, | |
| { | |
| "count/fg_chosen": 35.55555725097656, | |
| "count/fg_rejected": 7.764705657958984, | |
| "epoch": 0.7692307692307693, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -12.14694881439209, | |
| "fg_logps/policy_chosen": -5.938383102416992, | |
| "fg_logps/policy_rejected": -6.999124526977539, | |
| "fg_logps/reference_KL": -9.31165599822998, | |
| "fg_logps/reference_chosen": -5.441190242767334, | |
| "fg_logps/reference_rejected": -6.349566459655762, | |
| "fg_loss": 0.7159730792045593, | |
| "fg_rewards/chosen_sum": -1.1845917701721191, | |
| "fg_rewards/rejected_sum": -0.6632856726646423, | |
| "grad_norm": 29.30713270318622, | |
| "kl": 0.0, | |
| "learning_rate": 1.2884834663625997e-07, | |
| "logps/chosen": -351.3742959665698, | |
| "logps/rejected": -477.6983741554054, | |
| "loss": 0.4247, | |
| "rewards/chosen": 1.214830709058185, | |
| "rewards/margins": 3.774690938550372, | |
| "rewards/rejected": -2.5598602294921875, | |
| "step": 750 | |
| }, | |
| { | |
| "count/fg_chosen": 29.450000762939453, | |
| "count/fg_rejected": 8.941176414489746, | |
| "epoch": 0.7794871794871795, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.91418743133545, | |
| "fg_logps/policy_chosen": -6.500802516937256, | |
| "fg_logps/policy_rejected": -7.696014404296875, | |
| "fg_logps/reference_KL": -10.872803688049316, | |
| "fg_logps/reference_chosen": -6.060533046722412, | |
| "fg_logps/reference_rejected": -6.9095940589904785, | |
| "fg_loss": 0.8821809887886047, | |
| "fg_rewards/chosen_sum": -1.222095012664795, | |
| "fg_rewards/rejected_sum": -0.8427726030349731, | |
| "grad_norm": 23.409794702847652, | |
| "kl": 0.0, | |
| "learning_rate": 1.2314709236031927e-07, | |
| "logps/chosen": -350.1331449468085, | |
| "logps/rejected": -399.28329190340907, | |
| "loss": 0.4556, | |
| "rewards/chosen": 1.079000432440575, | |
| "rewards/margins": 3.400327179525684, | |
| "rewards/rejected": -2.321326747085109, | |
| "step": 760 | |
| }, | |
| { | |
| "count/fg_chosen": 26.789474487304688, | |
| "count/fg_rejected": 7.0, | |
| "epoch": 0.7897435897435897, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -14.873922348022461, | |
| "fg_logps/policy_chosen": -7.308094024658203, | |
| "fg_logps/policy_rejected": -7.175009727478027, | |
| "fg_logps/reference_KL": -11.33347225189209, | |
| "fg_logps/reference_chosen": -6.172826766967773, | |
| "fg_logps/reference_rejected": -6.013812065124512, | |
| "fg_loss": 0.7144444584846497, | |
| "fg_rewards/chosen_sum": -2.5322296619415283, | |
| "fg_rewards/rejected_sum": -0.9527682065963745, | |
| "grad_norm": 39.24437423103846, | |
| "kl": 0.0, | |
| "learning_rate": 1.1744583808437855e-07, | |
| "logps/chosen": -331.80161458333333, | |
| "logps/rejected": -438.73373161764704, | |
| "loss": 0.4217, | |
| "rewards/chosen": 1.7163297526041668, | |
| "rewards/margins": 3.8777695360370714, | |
| "rewards/rejected": -2.1614397834329044, | |
| "step": 770 | |
| }, | |
| { | |
| "count/fg_chosen": 32.1875, | |
| "count/fg_rejected": 5.8125, | |
| "epoch": 0.8, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.304938316345215, | |
| "fg_logps/policy_chosen": -6.930516242980957, | |
| "fg_logps/policy_rejected": -7.694925785064697, | |
| "fg_logps/reference_KL": -10.620773315429688, | |
| "fg_logps/reference_chosen": -6.272948265075684, | |
| "fg_logps/reference_rejected": -6.997465133666992, | |
| "fg_loss": 0.7136435508728027, | |
| "fg_rewards/chosen_sum": -1.7926644086837769, | |
| "fg_rewards/rejected_sum": -0.3232946991920471, | |
| "grad_norm": 24.909110123805196, | |
| "kl": 0.0, | |
| "learning_rate": 1.1174458380843785e-07, | |
| "logps/chosen": -303.2761627906977, | |
| "logps/rejected": -392.0064400337838, | |
| "loss": 0.4265, | |
| "rewards/chosen": 1.246623904206032, | |
| "rewards/margins": 3.1565479731724597, | |
| "rewards/rejected": -1.9099240689664274, | |
| "step": 780 | |
| }, | |
| { | |
| "count/fg_chosen": 31.0, | |
| "count/fg_rejected": 6.0, | |
| "epoch": 0.8102564102564103, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.473762512207031, | |
| "fg_logps/policy_chosen": -6.058125972747803, | |
| "fg_logps/policy_rejected": -7.64716100692749, | |
| "fg_logps/reference_KL": -10.569419860839844, | |
| "fg_logps/reference_chosen": -5.636739730834961, | |
| "fg_logps/reference_rejected": -6.955509662628174, | |
| "fg_loss": 0.6918079257011414, | |
| "fg_rewards/chosen_sum": -0.8551385998725891, | |
| "fg_rewards/rejected_sum": -0.39899593591690063, | |
| "grad_norm": 22.682895979877173, | |
| "kl": 0.0, | |
| "learning_rate": 1.0604332953249714e-07, | |
| "logps/chosen": -329.2703077936747, | |
| "logps/rejected": -374.5271154626623, | |
| "loss": 0.4628, | |
| "rewards/chosen": 1.6247121052569653, | |
| "rewards/margins": 2.985569189456945, | |
| "rewards/rejected": -1.3608570841999796, | |
| "step": 790 | |
| }, | |
| { | |
| "count/fg_chosen": 32.46666717529297, | |
| "count/fg_rejected": 5.666666507720947, | |
| "epoch": 0.8205128205128205, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -14.543877601623535, | |
| "fg_logps/policy_chosen": -7.2420654296875, | |
| "fg_logps/policy_rejected": -8.499801635742188, | |
| "fg_logps/reference_KL": -11.280120849609375, | |
| "fg_logps/reference_chosen": -6.594120502471924, | |
| "fg_logps/reference_rejected": -7.241811275482178, | |
| "fg_loss": 0.6977981925010681, | |
| "fg_rewards/chosen_sum": -1.8107116222381592, | |
| "fg_rewards/rejected_sum": -0.6922832727432251, | |
| "grad_norm": 39.98049317260069, | |
| "kl": 0.0, | |
| "learning_rate": 1.0034207525655644e-07, | |
| "logps/chosen": -359.1409722222222, | |
| "logps/rejected": -515.9350446428572, | |
| "loss": 0.4043, | |
| "rewards/chosen": 1.5236521402994792, | |
| "rewards/margins": 3.4145729428245906, | |
| "rewards/rejected": -1.8909208025251116, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 0.8205128205128205, | |
| "eval_count/fg_chosen": 30.183246612548828, | |
| "eval_count/fg_rejected": 6.92391300201416, | |
| "eval_fg_kl": NaN, | |
| "eval_fg_logps/policy_KL": -14.794645309448242, | |
| "eval_fg_logps/policy_chosen": -6.733245849609375, | |
| "eval_fg_logps/policy_rejected": -8.626864433288574, | |
| "eval_fg_logps/reference_KL": -11.47359848022461, | |
| "eval_fg_logps/reference_chosen": -6.041894912719727, | |
| "eval_fg_logps/reference_rejected": -7.58065938949585, | |
| "eval_fg_loss": 0.762517511844635, | |
| "eval_fg_rewards/chosen_sum": -1.556026816368103, | |
| "eval_fg_rewards/rejected_sum": -0.9032577276229858, | |
| "eval_kl": 0.014131884090602398, | |
| "eval_logps/chosen": -336.04120131729667, | |
| "eval_logps/rejected": -406.1173232908459, | |
| "eval_loss": 0.41103363037109375, | |
| "eval_rewards/chosen": 1.7359535243503006, | |
| "eval_rewards/margins": 3.998709942730949, | |
| "eval_rewards/rejected": -2.262756418380649, | |
| "eval_runtime": 462.7715, | |
| "eval_samples_per_second": 3.745, | |
| "eval_steps_per_second": 0.938, | |
| "step": 800 | |
| }, | |
| { | |
| "count/fg_chosen": 25.733333587646484, | |
| "count/fg_rejected": 8.800000190734863, | |
| "epoch": 0.8307692307692308, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -14.521686553955078, | |
| "fg_logps/policy_chosen": -6.098317623138428, | |
| "fg_logps/policy_rejected": -7.37031888961792, | |
| "fg_logps/reference_KL": -11.138436317443848, | |
| "fg_logps/reference_chosen": -5.529090881347656, | |
| "fg_logps/reference_rejected": -6.567668437957764, | |
| "fg_loss": 0.7037224173545837, | |
| "fg_rewards/chosen_sum": -1.2442917823791504, | |
| "fg_rewards/rejected_sum": -0.9032351970672607, | |
| "grad_norm": 39.139843341578626, | |
| "kl": 0.0, | |
| "learning_rate": 9.464082098061574e-08, | |
| "logps/chosen": -351.8864535108025, | |
| "logps/rejected": -414.8218453322785, | |
| "loss": 0.4442, | |
| "rewards/chosen": 1.5498073248215665, | |
| "rewards/margins": 3.332525065809996, | |
| "rewards/rejected": -1.7827177409884296, | |
| "step": 810 | |
| }, | |
| { | |
| "count/fg_chosen": 29.53333282470703, | |
| "count/fg_rejected": 8.714285850524902, | |
| "epoch": 0.841025641025641, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -16.194475173950195, | |
| "fg_logps/policy_chosen": -7.472283840179443, | |
| "fg_logps/policy_rejected": -8.897085189819336, | |
| "fg_logps/reference_KL": -12.283650398254395, | |
| "fg_logps/reference_chosen": -6.056351184844971, | |
| "fg_logps/reference_rejected": -7.35612154006958, | |
| "fg_loss": 0.8785532712936401, | |
| "fg_rewards/chosen_sum": -2.9767565727233887, | |
| "fg_rewards/rejected_sum": -1.322200059890747, | |
| "grad_norm": 18.084277972168177, | |
| "kl": 0.11083474010229111, | |
| "learning_rate": 8.893956670467502e-08, | |
| "logps/chosen": -357.466950491573, | |
| "logps/rejected": -436.3477937940141, | |
| "loss": 0.4174, | |
| "rewards/chosen": 1.0562572693556882, | |
| "rewards/margins": 4.643260735464843, | |
| "rewards/rejected": -3.587003466109155, | |
| "step": 820 | |
| }, | |
| { | |
| "count/fg_chosen": 38.94117736816406, | |
| "count/fg_rejected": 9.133333206176758, | |
| "epoch": 0.8512820512820513, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -16.23822021484375, | |
| "fg_logps/policy_chosen": -6.3059844970703125, | |
| "fg_logps/policy_rejected": -7.230159282684326, | |
| "fg_logps/reference_KL": -12.178247451782227, | |
| "fg_logps/reference_chosen": -5.708430290222168, | |
| "fg_logps/reference_rejected": -5.989579200744629, | |
| "fg_loss": 0.8925216794013977, | |
| "fg_rewards/chosen_sum": -1.6965184211730957, | |
| "fg_rewards/rejected_sum": -1.320400357246399, | |
| "grad_norm": 36.47201338586735, | |
| "kl": 0.0, | |
| "learning_rate": 8.323831242873432e-08, | |
| "logps/chosen": -357.6101471656977, | |
| "logps/rejected": -458.4434121621622, | |
| "loss": 0.4677, | |
| "rewards/chosen": 1.8875178847202034, | |
| "rewards/margins": 3.2487285789043776, | |
| "rewards/rejected": -1.3612106941841744, | |
| "step": 830 | |
| }, | |
| { | |
| "count/fg_chosen": 32.33333206176758, | |
| "count/fg_rejected": 6.941176414489746, | |
| "epoch": 0.8615384615384616, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.08393669128418, | |
| "fg_logps/policy_chosen": -6.780251502990723, | |
| "fg_logps/policy_rejected": -7.011376857757568, | |
| "fg_logps/reference_KL": -10.381501197814941, | |
| "fg_logps/reference_chosen": -6.354065418243408, | |
| "fg_logps/reference_rejected": -6.308195114135742, | |
| "fg_loss": 0.7612662315368652, | |
| "fg_rewards/chosen_sum": -1.2925324440002441, | |
| "fg_rewards/rejected_sum": -0.4022027552127838, | |
| "grad_norm": 32.781155840821306, | |
| "kl": 0.0, | |
| "learning_rate": 7.753705815279361e-08, | |
| "logps/chosen": -331.03251953125, | |
| "logps/rejected": -414.158447265625, | |
| "loss": 0.4681, | |
| "rewards/chosen": 1.1188287734985352, | |
| "rewards/margins": 3.2083324432373046, | |
| "rewards/rejected": -2.0895036697387694, | |
| "step": 840 | |
| }, | |
| { | |
| "count/fg_chosen": 31.047618865966797, | |
| "count/fg_rejected": 9.380952835083008, | |
| "epoch": 0.8717948717948718, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -14.173805236816406, | |
| "fg_logps/policy_chosen": -6.497308731079102, | |
| "fg_logps/policy_rejected": -8.013084411621094, | |
| "fg_logps/reference_KL": -10.995800018310547, | |
| "fg_logps/reference_chosen": -5.791131973266602, | |
| "fg_logps/reference_rejected": -7.026320457458496, | |
| "fg_loss": 0.8731069564819336, | |
| "fg_rewards/chosen_sum": -1.6361898183822632, | |
| "fg_rewards/rejected_sum": -1.0837599039077759, | |
| "grad_norm": 29.75869838237667, | |
| "kl": 0.0, | |
| "learning_rate": 7.183580387685291e-08, | |
| "logps/chosen": -385.02049512987014, | |
| "logps/rejected": -396.0473926957831, | |
| "loss": 0.4773, | |
| "rewards/chosen": 1.234356471470424, | |
| "rewards/margins": 3.495361367519464, | |
| "rewards/rejected": -2.26100489604904, | |
| "step": 850 | |
| }, | |
| { | |
| "count/fg_chosen": 30.14285659790039, | |
| "count/fg_rejected": 5.857142925262451, | |
| "epoch": 0.882051282051282, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -15.854009628295898, | |
| "fg_logps/policy_chosen": -6.418879508972168, | |
| "fg_logps/policy_rejected": -8.779900550842285, | |
| "fg_logps/reference_KL": -12.032855033874512, | |
| "fg_logps/reference_chosen": -5.604477405548096, | |
| "fg_logps/reference_rejected": -7.313387870788574, | |
| "fg_loss": 0.8508789539337158, | |
| "fg_rewards/chosen_sum": -1.4381144046783447, | |
| "fg_rewards/rejected_sum": -0.7955017685890198, | |
| "grad_norm": 25.28218871544972, | |
| "kl": 0.0, | |
| "learning_rate": 6.613454960091219e-08, | |
| "logps/chosen": -300.05953414351853, | |
| "logps/rejected": -383.09023931962025, | |
| "loss": 0.3998, | |
| "rewards/chosen": 1.808587156696084, | |
| "rewards/margins": 3.9609218286823378, | |
| "rewards/rejected": -2.152334671986254, | |
| "step": 860 | |
| }, | |
| { | |
| "count/fg_chosen": 28.25, | |
| "count/fg_rejected": 5.800000190734863, | |
| "epoch": 0.8923076923076924, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -14.163013458251953, | |
| "fg_logps/policy_chosen": -6.579830646514893, | |
| "fg_logps/policy_rejected": -7.748932361602783, | |
| "fg_logps/reference_KL": -10.930763244628906, | |
| "fg_logps/reference_chosen": -5.884364128112793, | |
| "fg_logps/reference_rejected": -6.846892356872559, | |
| "fg_loss": 0.7400026321411133, | |
| "fg_rewards/chosen_sum": -1.4189780950546265, | |
| "fg_rewards/rejected_sum": -0.8359920978546143, | |
| "grad_norm": 35.432818767338894, | |
| "kl": 0.0, | |
| "learning_rate": 6.043329532497149e-08, | |
| "logps/chosen": -343.70828125, | |
| "logps/rejected": -468.51098345588235, | |
| "loss": 0.3876, | |
| "rewards/chosen": 1.4050553385416666, | |
| "rewards/margins": 4.349431846469056, | |
| "rewards/rejected": -2.9443765079273896, | |
| "step": 870 | |
| }, | |
| { | |
| "count/fg_chosen": 23.538461685180664, | |
| "count/fg_rejected": 6.166666507720947, | |
| "epoch": 0.9025641025641026, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -15.849803924560547, | |
| "fg_logps/policy_chosen": -6.889503002166748, | |
| "fg_logps/policy_rejected": -9.813651084899902, | |
| "fg_logps/reference_KL": -11.724679946899414, | |
| "fg_logps/reference_chosen": -6.115569114685059, | |
| "fg_logps/reference_rejected": -8.466334342956543, | |
| "fg_loss": 0.923600435256958, | |
| "fg_rewards/chosen_sum": -1.7626667022705078, | |
| "fg_rewards/rejected_sum": -0.8649892210960388, | |
| "grad_norm": 28.479901425326158, | |
| "kl": 0.0, | |
| "learning_rate": 5.4732041049030787e-08, | |
| "logps/chosen": -310.8540810032895, | |
| "logps/rejected": -370.5546642485119, | |
| "loss": 0.3796, | |
| "rewards/chosen": 2.1148808127955387, | |
| "rewards/margins": 4.169318722603016, | |
| "rewards/rejected": -2.054437909807478, | |
| "step": 880 | |
| }, | |
| { | |
| "count/fg_chosen": 28.647058486938477, | |
| "count/fg_rejected": 7.214285850524902, | |
| "epoch": 0.9128205128205128, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -14.122632026672363, | |
| "fg_logps/policy_chosen": -6.575372219085693, | |
| "fg_logps/policy_rejected": -8.797600746154785, | |
| "fg_logps/reference_KL": -11.014878273010254, | |
| "fg_logps/reference_chosen": -6.147511005401611, | |
| "fg_logps/reference_rejected": -7.686254024505615, | |
| "fg_loss": 0.7267153263092041, | |
| "fg_rewards/chosen_sum": -1.0292545557022095, | |
| "fg_rewards/rejected_sum": -1.021606206893921, | |
| "grad_norm": 14.860543570560992, | |
| "kl": 0.0, | |
| "learning_rate": 4.9030786773090077e-08, | |
| "logps/chosen": -310.51392463235294, | |
| "logps/rejected": -438.26770833333336, | |
| "loss": 0.4036, | |
| "rewards/chosen": 1.7528151568244486, | |
| "rewards/margins": 4.661474010991116, | |
| "rewards/rejected": -2.9086588541666667, | |
| "step": 890 | |
| }, | |
| { | |
| "count/fg_chosen": 25.649999618530273, | |
| "count/fg_rejected": 5.099999904632568, | |
| "epoch": 0.9230769230769231, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -15.193872451782227, | |
| "fg_logps/policy_chosen": -6.921626091003418, | |
| "fg_logps/policy_rejected": -9.522343635559082, | |
| "fg_logps/reference_KL": -11.5450439453125, | |
| "fg_logps/reference_chosen": -6.295389175415039, | |
| "fg_logps/reference_rejected": -8.422341346740723, | |
| "fg_loss": 0.8100715279579163, | |
| "fg_rewards/chosen_sum": -1.2453607320785522, | |
| "fg_rewards/rejected_sum": -0.6394971609115601, | |
| "grad_norm": 26.313505637704687, | |
| "kl": 0.0, | |
| "learning_rate": 4.332953249714937e-08, | |
| "logps/chosen": -379.3786095727848, | |
| "logps/rejected": -469.6590470679012, | |
| "loss": 0.4082, | |
| "rewards/chosen": 1.5717159222952928, | |
| "rewards/margins": 4.730814041206847, | |
| "rewards/rejected": -3.1590981189115546, | |
| "step": 900 | |
| }, | |
| { | |
| "count/fg_chosen": 31.809524536132812, | |
| "count/fg_rejected": 7.949999809265137, | |
| "epoch": 0.9333333333333333, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -13.88550090789795, | |
| "fg_logps/policy_chosen": -6.575550556182861, | |
| "fg_logps/policy_rejected": -8.97218132019043, | |
| "fg_logps/reference_KL": -10.913046836853027, | |
| "fg_logps/reference_chosen": -5.786440372467041, | |
| "fg_logps/reference_rejected": -8.161312103271484, | |
| "fg_loss": 0.6918947696685791, | |
| "fg_rewards/chosen_sum": -1.8372831344604492, | |
| "fg_rewards/rejected_sum": -0.7708438038825989, | |
| "grad_norm": 30.704094667430244, | |
| "kl": 0.0, | |
| "learning_rate": 3.762827822120866e-08, | |
| "logps/chosen": -342.4310891544118, | |
| "logps/rejected": -465.26286458333334, | |
| "loss": 0.394, | |
| "rewards/chosen": 1.6974679385914522, | |
| "rewards/margins": 5.162705975700828, | |
| "rewards/rejected": -3.465238037109375, | |
| "step": 910 | |
| }, | |
| { | |
| "count/fg_chosen": 33.44444274902344, | |
| "count/fg_rejected": 7.05555534362793, | |
| "epoch": 0.9435897435897436, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -16.852989196777344, | |
| "fg_logps/policy_chosen": -8.116382598876953, | |
| "fg_logps/policy_rejected": -9.44908332824707, | |
| "fg_logps/reference_KL": -12.696809768676758, | |
| "fg_logps/reference_chosen": -6.711515426635742, | |
| "fg_logps/reference_rejected": -8.2290678024292, | |
| "fg_loss": 0.8823240995407104, | |
| "fg_rewards/chosen_sum": -2.763169527053833, | |
| "fg_rewards/rejected_sum": -0.838661789894104, | |
| "grad_norm": 28.750070922344392, | |
| "kl": 0.0, | |
| "learning_rate": 3.192702394526796e-08, | |
| "logps/chosen": -369.98503449675326, | |
| "logps/rejected": -378.85448042168673, | |
| "loss": 0.455, | |
| "rewards/chosen": 1.3270432113052963, | |
| "rewards/margins": 4.1537548429882785, | |
| "rewards/rejected": -2.826711631682982, | |
| "step": 920 | |
| }, | |
| { | |
| "count/fg_chosen": 31.0, | |
| "count/fg_rejected": 7.222222328186035, | |
| "epoch": 0.9538461538461539, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -16.319583892822266, | |
| "fg_logps/policy_chosen": -7.1412739753723145, | |
| "fg_logps/policy_rejected": -8.26430892944336, | |
| "fg_logps/reference_KL": -12.383834838867188, | |
| "fg_logps/reference_chosen": -6.26031494140625, | |
| "fg_logps/reference_rejected": -7.345766544342041, | |
| "fg_loss": 0.7640350461006165, | |
| "fg_rewards/chosen_sum": -1.9626283645629883, | |
| "fg_rewards/rejected_sum": -0.9573346376419067, | |
| "grad_norm": 30.27804248529551, | |
| "kl": 0.0, | |
| "learning_rate": 2.6225769669327253e-08, | |
| "logps/chosen": -328.4565281723485, | |
| "logps/rejected": -376.20595079787233, | |
| "loss": 0.3767, | |
| "rewards/chosen": 2.0332070552941524, | |
| "rewards/margins": 5.176523121305622, | |
| "rewards/rejected": -3.1433160660114696, | |
| "step": 930 | |
| }, | |
| { | |
| "count/fg_chosen": 30.625, | |
| "count/fg_rejected": 7.0, | |
| "epoch": 0.9641025641025641, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -15.706071853637695, | |
| "fg_logps/policy_chosen": -7.53517484664917, | |
| "fg_logps/policy_rejected": -10.4346284866333, | |
| "fg_logps/reference_KL": -11.768562316894531, | |
| "fg_logps/reference_chosen": -6.489566326141357, | |
| "fg_logps/reference_rejected": -9.170198440551758, | |
| "fg_loss": 0.7636561989784241, | |
| "fg_rewards/chosen_sum": -2.5118489265441895, | |
| "fg_rewards/rejected_sum": -0.7650282979011536, | |
| "grad_norm": 39.27702261397548, | |
| "kl": 0.0, | |
| "learning_rate": 2.0524515393386543e-08, | |
| "logps/chosen": -346.63032670454544, | |
| "logps/rejected": -424.40479103915663, | |
| "loss": 0.4212, | |
| "rewards/chosen": 1.459620389071378, | |
| "rewards/margins": 4.2732272947213215, | |
| "rewards/rejected": -2.8136069056499435, | |
| "step": 940 | |
| }, | |
| { | |
| "count/fg_chosen": 25.272727966308594, | |
| "count/fg_rejected": 4.363636493682861, | |
| "epoch": 0.9743589743589743, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -14.18252182006836, | |
| "fg_logps/policy_chosen": -6.842197895050049, | |
| "fg_logps/policy_rejected": -7.672128677368164, | |
| "fg_logps/reference_KL": -10.544868469238281, | |
| "fg_logps/reference_chosen": -5.763217449188232, | |
| "fg_logps/reference_rejected": -6.418917655944824, | |
| "fg_loss": 0.6513127088546753, | |
| "fg_rewards/chosen_sum": -1.7274693250656128, | |
| "fg_rewards/rejected_sum": -0.795689582824707, | |
| "grad_norm": 23.77562108897806, | |
| "kl": 0.0, | |
| "learning_rate": 1.4823261117445838e-08, | |
| "logps/chosen": -349.6753555689103, | |
| "logps/rejected": -403.2014100609756, | |
| "loss": 0.3828, | |
| "rewards/chosen": 1.3632464286608574, | |
| "rewards/margins": 3.730917743923815, | |
| "rewards/rejected": -2.367671315262957, | |
| "step": 950 | |
| }, | |
| { | |
| "count/fg_chosen": 26.649999618530273, | |
| "count/fg_rejected": 6.5, | |
| "epoch": 0.9846153846153847, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -15.6975736618042, | |
| "fg_logps/policy_chosen": -7.5320539474487305, | |
| "fg_logps/policy_rejected": -9.357258796691895, | |
| "fg_logps/reference_KL": -12.039111137390137, | |
| "fg_logps/reference_chosen": -6.5699005126953125, | |
| "fg_logps/reference_rejected": -8.148908615112305, | |
| "fg_loss": 0.6992577910423279, | |
| "fg_rewards/chosen_sum": -1.6473188400268555, | |
| "fg_rewards/rejected_sum": -0.9504286646842957, | |
| "grad_norm": 29.616218116706296, | |
| "kl": 0.0, | |
| "learning_rate": 9.122006841505132e-09, | |
| "logps/chosen": -366.2652652138158, | |
| "logps/rejected": -412.12672061011904, | |
| "loss": 0.4654, | |
| "rewards/chosen": 0.8081491369950143, | |
| "rewards/margins": 3.221180578819791, | |
| "rewards/rejected": -2.413031441824777, | |
| "step": 960 | |
| }, | |
| { | |
| "count/fg_chosen": 27.294116973876953, | |
| "count/fg_rejected": 6.294117450714111, | |
| "epoch": 0.9948717948717949, | |
| "fg_kl": NaN, | |
| "fg_logps/policy_KL": -15.349953651428223, | |
| "fg_logps/policy_chosen": -7.822242259979248, | |
| "fg_logps/policy_rejected": -8.426324844360352, | |
| "fg_logps/reference_KL": -11.580068588256836, | |
| "fg_logps/reference_chosen": -6.358785629272461, | |
| "fg_logps/reference_rejected": -7.00697135925293, | |
| "fg_loss": 0.8312911987304688, | |
| "fg_rewards/chosen_sum": -2.5030882358551025, | |
| "fg_rewards/rejected_sum": -0.9714083671569824, | |
| "grad_norm": 26.639106291790316, | |
| "kl": 0.0, | |
| "learning_rate": 3.420752565564424e-09, | |
| "logps/chosen": -419.9399809966216, | |
| "logps/rejected": -412.7228379360465, | |
| "loss": 0.4666, | |
| "rewards/chosen": 0.5730146459631018, | |
| "rewards/margins": 2.736655932413865, | |
| "rewards/rejected": -2.1636412864507633, | |
| "step": 970 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 975, | |
| "total_flos": 0.0, | |
| "train_loss": 0.45996271347388246, | |
| "train_runtime": 8430.3956, | |
| "train_samples_per_second": 1.85, | |
| "train_steps_per_second": 0.116 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 975, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 500, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": false, | |
| "should_training_stop": false | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |