{ "best_metric": null, "best_model_checkpoint": null, "epoch": 3.0, "eval_steps": 100, "global_step": 987, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "dpo_losses": 0.6931471824645996, "epoch": 0.0, "grad_norm": 2.1006424602315215, "learning_rate": 5.050505050505051e-09, "logits/chosen": -2.589059352874756, "logits/rejected": -2.6418559551239014, "logps/chosen": -212.42770385742188, "logps/rejected": -135.43048095703125, "loss": 0.6931, "positive_losses": 0.0, "rewards/accuracies": 0.0, "rewards/chosen": 0.0, "rewards/margins": 0.0, "rewards/margins_max": 0.0, "rewards/margins_min": 0.0, "rewards/margins_std": 0.0, "rewards/rejected": 0.0, "step": 1 }, { "dpo_losses": 0.6925780773162842, "epoch": 0.03, "grad_norm": 12.811305441645697, "learning_rate": 5.05050505050505e-08, "logits/chosen": -2.977125644683838, "logits/rejected": -2.981342315673828, "logps/chosen": -426.8561096191406, "logps/rejected": -265.4306640625, "loss": 0.6957, "positive_losses": 0.01630179025232792, "rewards/accuracies": 0.3888888955116272, "rewards/chosen": 0.0003335709043312818, "rewards/margins": 0.0011399724753573537, "rewards/margins_max": 0.002273110207170248, "rewards/margins_min": 6.834672603872605e-06, "rewards/margins_std": 0.0016024988144636154, "rewards/rejected": -0.0008064016001299024, "step": 10 }, { "dpo_losses": 0.6931944489479065, "epoch": 0.06, "grad_norm": 17.598175196981984, "learning_rate": 1.01010101010101e-07, "logits/chosen": -2.992300271987915, "logits/rejected": -2.8757376670837402, "logps/chosen": -376.1276550292969, "logps/rejected": -280.2401428222656, "loss": 0.7005, "positive_losses": 0.06999759376049042, "rewards/accuracies": 0.5, "rewards/chosen": 0.0003719997184816748, "rewards/margins": -9.284308907808736e-05, "rewards/margins_max": 0.0014239997835829854, "rewards/margins_min": -0.0016096860636025667, "rewards/margins_std": 0.0021451397333294153, "rewards/rejected": 0.000464842829387635, "step": 20 }, { "dpo_losses": 0.6936646699905396, "epoch": 0.09, "grad_norm": 16.284910292127993, "learning_rate": 1.5151515151515152e-07, "logits/chosen": -2.8964314460754395, "logits/rejected": -2.9219629764556885, "logps/chosen": -204.34725952148438, "logps/rejected": -278.52960205078125, "loss": 0.698, "positive_losses": 0.042078591883182526, "rewards/accuracies": 0.3499999940395355, "rewards/chosen": 0.0006226692348718643, "rewards/margins": -0.0010327090276405215, "rewards/margins_max": 0.0002273331192554906, "rewards/margins_min": -0.002292751334607601, "rewards/margins_std": 0.001781968749128282, "rewards/rejected": 0.001655378146097064, "step": 30 }, { "dpo_losses": 0.6926015615463257, "epoch": 0.12, "grad_norm": 7.600951496593335, "learning_rate": 2.02020202020202e-07, "logits/chosen": -2.841174602508545, "logits/rejected": -2.759981632232666, "logps/chosen": -320.43280029296875, "logps/rejected": -249.27462768554688, "loss": 0.6944, "positive_losses": 0.0006469726795330644, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": 0.0031745159067213535, "rewards/margins": 0.0010936097241938114, "rewards/margins_max": 0.0026608791667968035, "rewards/margins_min": -0.0004736593982670456, "rewards/margins_std": 0.0022164531983435154, "rewards/rejected": 0.0020809059496968985, "step": 40 }, { "dpo_losses": 0.6930263638496399, "epoch": 0.15, "grad_norm": 9.105946335609067, "learning_rate": 2.525252525252525e-07, "logits/chosen": -2.829671859741211, "logits/rejected": -2.848930835723877, "logps/chosen": -227.5389404296875, "logps/rejected": -240.80026245117188, "loss": 0.694, "positive_losses": 0.0, "rewards/accuracies": 0.5, "rewards/chosen": 0.004696831107139587, "rewards/margins": 0.00024281651712954044, "rewards/margins_max": 0.0018194876611232758, "rewards/margins_min": -0.0013338546268641949, "rewards/margins_std": 0.0022297496907413006, "rewards/rejected": 0.004454014357179403, "step": 50 }, { "dpo_losses": 0.6927760243415833, "epoch": 0.18, "grad_norm": 1.9982033646829467, "learning_rate": 3.0303030303030305e-07, "logits/chosen": -2.807051420211792, "logits/rejected": -2.722658157348633, "logps/chosen": -333.5701599121094, "logps/rejected": -296.24591064453125, "loss": 0.6935, "positive_losses": 0.01206359826028347, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": 0.007529796566814184, "rewards/margins": 0.0007501507643610239, "rewards/margins_max": 0.004166206810623407, "rewards/margins_min": -0.002665905049070716, "rewards/margins_std": 0.004831031896173954, "rewards/rejected": 0.006779646035283804, "step": 60 }, { "dpo_losses": 0.692486584186554, "epoch": 0.21, "grad_norm": 9.583384726566226, "learning_rate": 3.535353535353535e-07, "logits/chosen": -2.6647205352783203, "logits/rejected": -2.6472456455230713, "logps/chosen": -305.70159912109375, "logps/rejected": -241.12240600585938, "loss": 0.6938, "positive_losses": 0.006031799130141735, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.007261182181537151, "rewards/margins": 0.0013255949597805738, "rewards/margins_max": 0.003496175166219473, "rewards/margins_min": -0.0008449858287349343, "rewards/margins_std": 0.0030696645844727755, "rewards/rejected": 0.0059355879202485085, "step": 70 }, { "dpo_losses": 0.6925214529037476, "epoch": 0.24, "grad_norm": 2.10104329669103, "learning_rate": 4.04040404040404e-07, "logits/chosen": -2.965867280960083, "logits/rejected": -2.8706986904144287, "logps/chosen": -324.9981689453125, "logps/rejected": -331.00262451171875, "loss": 0.6938, "positive_losses": 0.0, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.009126158431172371, "rewards/margins": 0.0012557277223095298, "rewards/margins_max": 0.0036045038141310215, "rewards/margins_min": -0.0010930480202659965, "rewards/margins_std": 0.0033216706942766905, "rewards/rejected": 0.007870429195463657, "step": 80 }, { "dpo_losses": 0.6932989358901978, "epoch": 0.27, "grad_norm": 1.9836450871378364, "learning_rate": 4.545454545454545e-07, "logits/chosen": -2.7672338485717773, "logits/rejected": -2.6813530921936035, "logps/chosen": -280.62030029296875, "logps/rejected": -277.8973388671875, "loss": 0.6933, "positive_losses": 0.0020202635787427425, "rewards/accuracies": 0.550000011920929, "rewards/chosen": 0.007793852128088474, "rewards/margins": -0.00029996776720508933, "rewards/margins_max": 0.0021644725929945707, "rewards/margins_min": -0.0027644080109894276, "rewards/margins_std": 0.003485244931653142, "rewards/rejected": 0.008093819953501225, "step": 90 }, { "dpo_losses": 0.6919553279876709, "epoch": 0.3, "grad_norm": 1.713342030808654, "learning_rate": 4.99998435471878e-07, "logits/chosen": -2.971386432647705, "logits/rejected": -2.915938377380371, "logps/chosen": -269.40509033203125, "logps/rejected": -256.8268737792969, "loss": 0.6932, "positive_losses": 0.0, "rewards/accuracies": 0.75, "rewards/chosen": 0.008469906635582447, "rewards/margins": 0.002389331813901663, "rewards/margins_max": 0.004788221325725317, "rewards/margins_min": -9.557651537761558e-06, "rewards/margins_std": 0.0033925422467291355, "rewards/rejected": 0.006080574356019497, "step": 100 }, { "epoch": 0.3, "eval_dpo_losses": 0.6925311088562012, "eval_logits/chosen": -2.8593878746032715, "eval_logits/rejected": -2.8141348361968994, "eval_logps/chosen": -284.26416015625, "eval_logps/rejected": -258.3495178222656, "eval_loss": 0.6929529905319214, "eval_positive_losses": 0.0051858690567314625, "eval_rewards/accuracies": 0.579365074634552, "eval_rewards/chosen": 0.009570538066327572, "eval_rewards/margins": 0.0012392366770654917, "eval_rewards/margins_max": 0.006621083710342646, "eval_rewards/margins_min": -0.003959333524107933, "eval_rewards/margins_std": 0.004715622402727604, "eval_rewards/rejected": 0.00833130069077015, "eval_runtime": 282.6491, "eval_samples_per_second": 7.076, "eval_steps_per_second": 0.223, "step": 100 }, { "dpo_losses": 0.6927061676979065, "epoch": 0.33, "grad_norm": 2.1416302847338655, "learning_rate": 4.998107157902955e-07, "logits/chosen": -2.841651201248169, "logits/rejected": -2.74540638923645, "logps/chosen": -310.16998291015625, "logps/rejected": -254.25537109375, "loss": 0.6931, "positive_losses": 0.009991454891860485, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": 0.009014981798827648, "rewards/margins": 0.0008873892948031425, "rewards/margins_max": 0.0038191110361367464, "rewards/margins_min": -0.0020443324465304613, "rewards/margins_std": 0.004146079998463392, "rewards/rejected": 0.008127592504024506, "step": 110 }, { "dpo_losses": 0.6933146715164185, "epoch": 0.36, "grad_norm": 2.01446405141137, "learning_rate": 4.993103596812268e-07, "logits/chosen": -2.7760932445526123, "logits/rejected": -2.853368043899536, "logps/chosen": -241.74014282226562, "logps/rejected": -251.635009765625, "loss": 0.6928, "positive_losses": 0.0, "rewards/accuracies": 0.5, "rewards/chosen": 0.011283518746495247, "rewards/margins": -0.00032261465094052255, "rewards/margins_max": 0.0035319500602781773, "rewards/margins_min": -0.00417717918753624, "rewards/margins_std": 0.00545117724686861, "rewards/rejected": 0.011606132611632347, "step": 120 }, { "dpo_losses": 0.6912622451782227, "epoch": 0.4, "grad_norm": 1.6295192671904115, "learning_rate": 4.984979933369709e-07, "logits/chosen": -2.9324135780334473, "logits/rejected": -2.867285966873169, "logps/chosen": -442.0322265625, "logps/rejected": -269.4384765625, "loss": 0.6928, "positive_losses": 0.0, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.01560944877564907, "rewards/margins": 0.0037853687535971403, "rewards/margins_max": 0.007000140845775604, "rewards/margins_min": 0.0005705966614186764, "rewards/margins_std": 0.004546374548226595, "rewards/rejected": 0.011824080720543861, "step": 130 }, { "dpo_losses": 0.6920586824417114, "epoch": 0.43, "grad_norm": 1.9106724203537326, "learning_rate": 4.973746334285336e-07, "logits/chosen": -2.803515911102295, "logits/rejected": -2.8981337547302246, "logps/chosen": -340.9748229980469, "logps/rejected": -325.1402282714844, "loss": 0.6917, "positive_losses": 0.0, "rewards/accuracies": 0.5, "rewards/chosen": 0.01538917887955904, "rewards/margins": 0.002193502150475979, "rewards/margins_max": 0.005477132275700569, "rewards/margins_min": -0.001090128207579255, "rewards/margins_std": 0.004643755033612251, "rewards/rejected": 0.013195676729083061, "step": 140 }, { "dpo_losses": 0.6921483278274536, "epoch": 0.46, "grad_norm": 1.4481594604817483, "learning_rate": 4.959416858332709e-07, "logits/chosen": -2.855631113052368, "logits/rejected": -2.820399045944214, "logps/chosen": -271.1258544921875, "logps/rejected": -282.1231689453125, "loss": 0.6932, "positive_losses": 0.0, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.014782702550292015, "rewards/margins": 0.002010155702009797, "rewards/margins_max": 0.00527102081105113, "rewards/margins_min": -0.001250709407031536, "rewards/margins_std": 0.004611559212207794, "rewards/rejected": 0.012772548012435436, "step": 150 }, { "dpo_losses": 0.6921746730804443, "epoch": 0.49, "grad_norm": 2.122964894274462, "learning_rate": 4.942009438754421e-07, "logits/chosen": -2.8902671337127686, "logits/rejected": -2.875211238861084, "logps/chosen": -210.3759765625, "logps/rejected": -202.38551330566406, "loss": 0.6939, "positive_losses": 0.0, "rewards/accuracies": 0.44999998807907104, "rewards/chosen": 0.014994035474956036, "rewards/margins": 0.0019599986262619495, "rewards/margins_max": 0.006691869348287582, "rewards/margins_min": -0.0027718725614249706, "rewards/margins_std": 0.006691875867545605, "rewards/rejected": 0.013034040108323097, "step": 160 }, { "dpo_losses": 0.690701961517334, "epoch": 0.52, "grad_norm": 1.585863985509441, "learning_rate": 4.921545860818778e-07, "logits/chosen": -2.8698291778564453, "logits/rejected": -2.769212484359741, "logps/chosen": -274.9241943359375, "logps/rejected": -276.3659973144531, "loss": 0.6909, "positive_losses": 0.0, "rewards/accuracies": 0.75, "rewards/chosen": 0.019051264971494675, "rewards/margins": 0.004921003244817257, "rewards/margins_max": 0.009086159989237785, "rewards/margins_min": 0.0007558457436971366, "rewards/margins_std": 0.005890422035008669, "rewards/rejected": 0.014130261726677418, "step": 170 }, { "dpo_losses": 0.692548394203186, "epoch": 0.55, "grad_norm": 8.258786695285371, "learning_rate": 4.898051734555674e-07, "logits/chosen": -2.8515067100524902, "logits/rejected": -2.865828037261963, "logps/chosen": -193.68429565429688, "logps/rejected": -217.70962524414062, "loss": 0.6921, "positive_losses": 0.0, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": 0.016250569373369217, "rewards/margins": 0.0012078474974259734, "rewards/margins_max": 0.0032145425211638212, "rewards/margins_min": -0.0007988469442352653, "rewards/margins_std": 0.002837894717231393, "rewards/rejected": 0.01504272036254406, "step": 180 }, { "dpo_losses": 0.6910722255706787, "epoch": 0.58, "grad_norm": 9.556509896855202, "learning_rate": 4.871556462705816e-07, "logits/chosen": -2.663051128387451, "logits/rejected": -2.8675954341888428, "logps/chosen": -174.3250732421875, "logps/rejected": -239.2481231689453, "loss": 0.692, "positive_losses": 0.0, "rewards/accuracies": 0.5, "rewards/chosen": 0.020512599498033524, "rewards/margins": 0.004181156400591135, "rewards/margins_max": 0.009443456307053566, "rewards/margins_min": -0.0010811425745487213, "rewards/margins_std": 0.007442014757543802, "rewards/rejected": 0.016331443563103676, "step": 190 }, { "dpo_losses": 0.6924042701721191, "epoch": 0.61, "grad_norm": 2.1686233219670368, "learning_rate": 4.842093203923387e-07, "logits/chosen": -2.906074047088623, "logits/rejected": -2.8612585067749023, "logps/chosen": -219.8831787109375, "logps/rejected": -310.6004638671875, "loss": 0.6919, "positive_losses": 0.0378265380859375, "rewards/accuracies": 0.5, "rewards/chosen": 0.02058028057217598, "rewards/margins": 0.0015535615384578705, "rewards/margins_max": 0.009957343339920044, "rewards/margins_min": -0.006850218866020441, "rewards/margins_std": 0.011884740553796291, "rewards/rejected": 0.01902671717107296, "step": 200 }, { "epoch": 0.61, "eval_dpo_losses": 0.6904077529907227, "eval_logits/chosen": -2.858304977416992, "eval_logits/rejected": -2.813277244567871, "eval_logps/chosen": -282.7998962402344, "eval_logps/rejected": -257.31414794921875, "eval_loss": 0.6915329694747925, "eval_positive_losses": 0.01193891279399395, "eval_rewards/accuracies": 0.6666666865348816, "eval_rewards/chosen": 0.024213315919041634, "eval_rewards/margins": 0.0055284094996750355, "eval_rewards/margins_max": 0.019484752789139748, "eval_rewards/margins_min": -0.008329018950462341, "eval_rewards/margins_std": 0.012371117249131203, "eval_rewards/rejected": 0.018684905022382736, "eval_runtime": 282.1184, "eval_samples_per_second": 7.089, "eval_steps_per_second": 0.223, "step": 200 }, { "dpo_losses": 0.6858643293380737, "epoch": 0.64, "grad_norm": 1.962735267678132, "learning_rate": 4.809698831278217e-07, "logits/chosen": -2.9653470516204834, "logits/rejected": -2.9085001945495605, "logps/chosen": -375.43804931640625, "logps/rejected": -293.70599365234375, "loss": 0.6913, "positive_losses": 0.0, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 0.032657068222761154, "rewards/margins": 0.014679856598377228, "rewards/margins_max": 0.026673411950469017, "rewards/margins_min": 0.002686301711946726, "rewards/margins_std": 0.01696144789457321, "rewards/rejected": 0.017977211624383926, "step": 210 }, { "dpo_losses": 0.6917377710342407, "epoch": 0.67, "grad_norm": 1.9665123469481351, "learning_rate": 4.774413886109386e-07, "logits/chosen": -2.9416866302490234, "logits/rejected": -2.848280429840088, "logps/chosen": -281.5489807128906, "logps/rejected": -228.3254852294922, "loss": 0.6904, "positive_losses": 0.0, "rewards/accuracies": 0.550000011920929, "rewards/chosen": 0.024796273559331894, "rewards/margins": 0.002851649885997176, "rewards/margins_max": 0.00997900776565075, "rewards/margins_min": -0.004275707062333822, "rewards/margins_std": 0.010079605504870415, "rewards/rejected": 0.021944623440504074, "step": 220 }, { "dpo_losses": 0.6900176405906677, "epoch": 0.7, "grad_norm": 8.719769818128936, "learning_rate": 4.736282527288008e-07, "logits/chosen": -2.9049994945526123, "logits/rejected": -2.930379629135132, "logps/chosen": -204.6458740234375, "logps/rejected": -201.35501098632812, "loss": 0.6932, "positive_losses": 0.0, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.02477971278131008, "rewards/margins": 0.006342197768390179, "rewards/margins_max": 0.015214154496788979, "rewards/margins_min": -0.0025297594256699085, "rewards/margins_std": 0.012546841986477375, "rewards/rejected": 0.018437515944242477, "step": 230 }, { "dpo_losses": 0.6900538802146912, "epoch": 0.73, "grad_norm": 2.0583669693189552, "learning_rate": 4.6953524759527053e-07, "logits/chosen": -3.0629992485046387, "logits/rejected": -2.9669101238250732, "logps/chosen": -341.70391845703125, "logps/rejected": -327.5718078613281, "loss": 0.6911, "positive_losses": 0.0, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": 0.029439503327012062, "rewards/margins": 0.006293328013271093, "rewards/margins_max": 0.019671425223350525, "rewards/margins_min": -0.0070847696624696255, "rewards/margins_std": 0.018919486552476883, "rewards/rejected": 0.023146172985434532, "step": 240 }, { "dpo_losses": 0.6918362379074097, "epoch": 0.76, "grad_norm": 1.9864876382070107, "learning_rate": 4.6516749557869185e-07, "logits/chosen": -2.9660696983337402, "logits/rejected": -2.871572494506836, "logps/chosen": -272.2618408203125, "logps/rejected": -243.72860717773438, "loss": 0.6926, "positive_losses": 0.0, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.030278582125902176, "rewards/margins": 0.002722159493714571, "rewards/margins_max": 0.014538141898810863, "rewards/margins_min": -0.009093823842704296, "rewards/margins_std": 0.016710322350263596, "rewards/rejected": 0.027556419372558594, "step": 250 }, { "dpo_losses": 0.6899689435958862, "epoch": 0.79, "grad_norm": 2.12838016274278, "learning_rate": 4.6053046289128194e-07, "logits/chosen": -2.956109046936035, "logits/rejected": -2.9426538944244385, "logps/chosen": -241.18115234375, "logps/rejected": -204.5635528564453, "loss": 0.6896, "positive_losses": 0.00806427001953125, "rewards/accuracies": 0.550000011920929, "rewards/chosen": 0.029516180977225304, "rewards/margins": 0.006419041194021702, "rewards/margins_max": 0.014670031145215034, "rewards/margins_min": -0.0018319508526474237, "rewards/margins_std": 0.011668664403259754, "rewards/rejected": 0.023097142577171326, "step": 260 }, { "dpo_losses": 0.6896145343780518, "epoch": 0.82, "grad_norm": 2.012252233192398, "learning_rate": 4.5562995274820283e-07, "logits/chosen": -2.819646120071411, "logits/rejected": -2.845240354537964, "logps/chosen": -307.28582763671875, "logps/rejected": -313.50897216796875, "loss": 0.6913, "positive_losses": 0.06691131740808487, "rewards/accuracies": 0.550000011920929, "rewards/chosen": 0.025536488741636276, "rewards/margins": 0.007166997995227575, "rewards/margins_max": 0.01585778035223484, "rewards/margins_min": -0.0015237818006426096, "rewards/margins_std": 0.012290618382394314, "rewards/rejected": 0.018369488418102264, "step": 270 }, { "dpo_losses": 0.6887096166610718, "epoch": 0.85, "grad_norm": 2.181123977999317, "learning_rate": 4.5047209810487705e-07, "logits/chosen": -2.971086025238037, "logits/rejected": -2.9251911640167236, "logps/chosen": -351.91864013671875, "logps/rejected": -323.68658447265625, "loss": 0.6897, "positive_losses": 0.03092041052877903, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.04124215617775917, "rewards/margins": 0.009057796560227871, "rewards/margins_max": 0.017794614657759666, "rewards/margins_min": 0.00032097622170113027, "rewards/margins_std": 0.012355729006230831, "rewards/rejected": 0.032184358686208725, "step": 280 }, { "dpo_losses": 0.6910039186477661, "epoch": 0.88, "grad_norm": 1.7264365586137154, "learning_rate": 4.450633539816351e-07, "logits/chosen": -2.669238567352295, "logits/rejected": -2.6185073852539062, "logps/chosen": -229.40542602539062, "logps/rejected": -268.8614501953125, "loss": 0.6898, "positive_losses": 0.0014183043967932463, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": 0.029668454080820084, "rewards/margins": 0.00436769612133503, "rewards/margins_max": 0.01580682210624218, "rewards/margins_min": -0.007071429397910833, "rewards/margins_std": 0.01617736741900444, "rewards/rejected": 0.025300756096839905, "step": 290 }, { "dpo_losses": 0.688437819480896, "epoch": 0.91, "grad_norm": 7.040817600804112, "learning_rate": 4.394104893853007e-07, "logits/chosen": -3.0286426544189453, "logits/rejected": -2.9285566806793213, "logps/chosen": -266.41265869140625, "logps/rejected": -276.48675537109375, "loss": 0.6903, "positive_losses": 0.08214263617992401, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.037517085671424866, "rewards/margins": 0.009616566821932793, "rewards/margins_max": 0.027836177498102188, "rewards/margins_min": -0.008603041991591454, "rewards/margins_std": 0.025766421109437943, "rewards/rejected": 0.027900516986846924, "step": 300 }, { "epoch": 0.91, "eval_dpo_losses": 0.6876209378242493, "eval_logits/chosen": -2.853738784790039, "eval_logits/rejected": -2.8086090087890625, "eval_logps/chosen": -281.2695007324219, "eval_logps/rejected": -256.35443115234375, "eval_loss": 0.6898627877235413, "eval_positive_losses": 0.016462840139865875, "eval_rewards/accuracies": 0.6666666865348816, "eval_rewards/chosen": 0.03951713442802429, "eval_rewards/margins": 0.011234736070036888, "eval_rewards/margins_max": 0.0378512442111969, "eval_rewards/margins_min": -0.014283931814134121, "eval_rewards/margins_std": 0.023247137665748596, "eval_rewards/rejected": 0.028282402083277702, "eval_runtime": 281.1088, "eval_samples_per_second": 7.115, "eval_steps_per_second": 0.224, "step": 300 }, { "dpo_losses": 0.6858936548233032, "epoch": 0.94, "grad_norm": 1.534301986044828, "learning_rate": 4.33520578837825e-07, "logits/chosen": -2.7751994132995605, "logits/rejected": -2.767688035964966, "logps/chosen": -286.08197021484375, "logps/rejected": -240.5784912109375, "loss": 0.6894, "positive_losses": 0.0, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.04037940502166748, "rewards/margins": 0.014671111479401588, "rewards/margins_max": 0.025094876065850258, "rewards/margins_min": 0.004247347824275494, "rewards/margins_std": 0.014741426333785057, "rewards/rejected": 0.02570829726755619, "step": 310 }, { "dpo_losses": 0.6850165128707886, "epoch": 0.97, "grad_norm": 1.9190817282141555, "learning_rate": 4.2740099352256954e-07, "logits/chosen": -2.9025392532348633, "logits/rejected": -2.907529592514038, "logps/chosen": -202.9761505126953, "logps/rejected": -240.7790985107422, "loss": 0.6868, "positive_losses": 0.0, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 0.04233746603131294, "rewards/margins": 0.016391225159168243, "rewards/margins_max": 0.023528441786766052, "rewards/margins_min": 0.00925400760024786, "rewards/margins_std": 0.010093549266457558, "rewards/rejected": 0.02594623900949955, "step": 320 }, { "dpo_losses": 0.6829380989074707, "epoch": 1.0, "grad_norm": 2.017378170300957, "learning_rate": 4.2105939205932005e-07, "logits/chosen": -2.9684784412384033, "logits/rejected": -2.8465845584869385, "logps/chosen": -384.70379638671875, "logps/rejected": -371.2561950683594, "loss": 0.6893, "positive_losses": 0.0, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 0.056094102561473846, "rewards/margins": 0.02067440189421177, "rewards/margins_max": 0.03426554799079895, "rewards/margins_min": 0.007083254866302013, "rewards/margins_std": 0.019220784306526184, "rewards/rejected": 0.03541969880461693, "step": 330 }, { "dpo_losses": 0.6854172348976135, "epoch": 1.03, "grad_norm": 2.002276198207298, "learning_rate": 4.145037109195757e-07, "logits/chosen": -2.9032974243164062, "logits/rejected": -2.9131686687469482, "logps/chosen": -265.29217529296875, "logps/rejected": -192.96426391601562, "loss": 0.6866, "positive_losses": 0.01656951941549778, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.044899724423885345, "rewards/margins": 0.01578436978161335, "rewards/margins_max": 0.03546427935361862, "rewards/margins_min": -0.003895541187375784, "rewards/margins_std": 0.02783159911632538, "rewards/rejected": 0.029115352779626846, "step": 340 }, { "dpo_losses": 0.6908459067344666, "epoch": 1.06, "grad_norm": 1.847200301127122, "learning_rate": 4.077421544941079e-07, "logits/chosen": -2.8273587226867676, "logits/rejected": -2.8343966007232666, "logps/chosen": -225.12521362304688, "logps/rejected": -211.3616180419922, "loss": 0.6867, "positive_losses": 0.089324951171875, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": 0.038552217185497284, "rewards/margins": 0.004919026046991348, "rewards/margins_max": 0.021458707749843597, "rewards/margins_min": -0.011620650999248028, "rewards/margins_std": 0.0233906377106905, "rewards/rejected": 0.03363319858908653, "step": 350 }, { "dpo_losses": 0.6812031865119934, "epoch": 1.09, "grad_norm": 1.9045252266933426, "learning_rate": 4.0078318482522114e-07, "logits/chosen": -2.8550143241882324, "logits/rejected": -2.851294755935669, "logps/chosen": -236.28811645507812, "logps/rejected": -254.1648406982422, "loss": 0.6854, "positive_losses": 0.0, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.05590778589248657, "rewards/margins": 0.024477163329720497, "rewards/margins_max": 0.046524226665496826, "rewards/margins_min": 0.0024301104713231325, "rewards/margins_std": 0.031179243698716164, "rewards/rejected": 0.031430620700120926, "step": 360 }, { "dpo_losses": 0.6793321371078491, "epoch": 1.12, "grad_norm": 2.2246778576418613, "learning_rate": 3.9363551101656356e-07, "logits/chosen": -2.8856711387634277, "logits/rejected": -2.7337353229522705, "logps/chosen": -360.27142333984375, "logps/rejected": -211.5453338623047, "loss": 0.6838, "positive_losses": 0.0, "rewards/accuracies": 0.75, "rewards/chosen": 0.05444517731666565, "rewards/margins": 0.02811727486550808, "rewards/margins_max": 0.04928036779165268, "rewards/margins_min": 0.006954180542379618, "rewards/margins_std": 0.029929134994745255, "rewards/rejected": 0.02632790245115757, "step": 370 }, { "dpo_losses": 0.6877564787864685, "epoch": 1.16, "grad_norm": 2.093111994313669, "learning_rate": 3.8630807833374277e-07, "logits/chosen": -2.9553494453430176, "logits/rejected": -2.8332180976867676, "logps/chosen": -270.87994384765625, "logps/rejected": -317.43060302734375, "loss": 0.6851, "positive_losses": 0.0, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.052814941853284836, "rewards/margins": 0.011218393221497536, "rewards/margins_max": 0.03882121667265892, "rewards/margins_min": -0.01638443022966385, "rewards/margins_std": 0.039036285132169724, "rewards/rejected": 0.04159654304385185, "step": 380 }, { "dpo_losses": 0.6842631101608276, "epoch": 1.19, "grad_norm": 1.9478894060100704, "learning_rate": 3.7881005700938627e-07, "logits/chosen": -2.883104085922241, "logits/rejected": -2.9553048610687256, "logps/chosen": -286.1315002441406, "logps/rejected": -273.46533203125, "loss": 0.6845, "positive_losses": 0.0, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.061849962919950485, "rewards/margins": 0.01832859218120575, "rewards/margins_max": 0.04364844784140587, "rewards/margins_min": -0.006991264875978231, "rewards/margins_std": 0.03580768406391144, "rewards/rejected": 0.043521374464035034, "step": 390 }, { "dpo_losses": 0.6858700513839722, "epoch": 1.22, "grad_norm": 2.027188274912056, "learning_rate": 3.711508307666567e-07, "logits/chosen": -2.8832974433898926, "logits/rejected": -2.911393404006958, "logps/chosen": -202.76405334472656, "logps/rejected": -219.851318359375, "loss": 0.6832, "positive_losses": 0.0, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.053437333554029465, "rewards/margins": 0.014844384975731373, "rewards/margins_max": 0.03350861743092537, "rewards/margins_min": -0.0038198456168174744, "rewards/margins_std": 0.02639520727097988, "rewards/rejected": 0.038592949509620667, "step": 400 }, { "epoch": 1.22, "eval_dpo_losses": 0.6846545934677124, "eval_logits/chosen": -2.850677490234375, "eval_logits/rejected": -2.8056881427764893, "eval_logps/chosen": -279.9754638671875, "eval_logps/rejected": -255.6741485595703, "eval_loss": 0.6891601085662842, "eval_positive_losses": 0.030417578294873238, "eval_rewards/accuracies": 0.7023809552192688, "eval_rewards/chosen": 0.052457116544246674, "eval_rewards/margins": 0.01737220212817192, "eval_rewards/margins_max": 0.05574695020914078, "eval_rewards/margins_min": -0.019588936120271683, "eval_rewards/margins_std": 0.03368803858757019, "eval_rewards/rejected": 0.03508492186665535, "eval_runtime": 281.5128, "eval_samples_per_second": 7.104, "eval_steps_per_second": 0.224, "step": 400 }, { "dpo_losses": 0.6754962205886841, "epoch": 1.25, "grad_norm": 1.8539817875046798, "learning_rate": 3.6333998507558596e-07, "logits/chosen": -2.7948994636535645, "logits/rejected": -2.838921070098877, "logps/chosen": -243.80331420898438, "logps/rejected": -267.6720275878906, "loss": 0.6841, "positive_losses": 0.0, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.061250943690538406, "rewards/margins": 0.03602340444922447, "rewards/margins_max": 0.05897093936800957, "rewards/margins_min": 0.013075867667794228, "rewards/margins_std": 0.032452717423439026, "rewards/rejected": 0.025227541103959084, "step": 410 }, { "dpo_losses": 0.6807809472084045, "epoch": 1.28, "grad_norm": 6.364237058452619, "learning_rate": 3.5538729515692354e-07, "logits/chosen": -2.943338394165039, "logits/rejected": -2.7816109657287598, "logps/chosen": -335.1874084472656, "logps/rejected": -279.122802734375, "loss": 0.6895, "positive_losses": 0.0, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.06752719730138779, "rewards/margins": 0.025191819295287132, "rewards/margins_max": 0.04893922060728073, "rewards/margins_min": 0.001444423571228981, "rewards/margins_std": 0.03358389437198639, "rewards/rejected": 0.04233537241816521, "step": 420 }, { "dpo_losses": 0.6775410175323486, "epoch": 1.31, "grad_norm": 2.1288430860755887, "learning_rate": 3.4730271374851453e-07, "logits/chosen": -2.8643321990966797, "logits/rejected": -2.8559885025024414, "logps/chosen": -335.872802734375, "logps/rejected": -249.48501586914062, "loss": 0.6787, "positive_losses": 0.0, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.06267213821411133, "rewards/margins": 0.03193260729312897, "rewards/margins_max": 0.05750267580151558, "rewards/margins_min": 0.006362536456435919, "rewards/margins_std": 0.03616153448820114, "rewards/rejected": 0.03073953464627266, "step": 430 }, { "dpo_losses": 0.6760178208351135, "epoch": 1.34, "grad_norm": 2.302278973535758, "learning_rate": 3.390963586495142e-07, "logits/chosen": -2.7527740001678467, "logits/rejected": -2.7448365688323975, "logps/chosen": -291.7049560546875, "logps/rejected": -197.5590362548828, "loss": 0.6836, "positive_losses": 0.0, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.06701546162366867, "rewards/margins": 0.03498576581478119, "rewards/margins_max": 0.05517752841114998, "rewards/margins_min": 0.01479400135576725, "rewards/margins_std": 0.02855546772480011, "rewards/rejected": 0.03202969953417778, "step": 440 }, { "dpo_losses": 0.6835552453994751, "epoch": 1.37, "grad_norm": 7.531724083885525, "learning_rate": 3.3077850005803125e-07, "logits/chosen": -3.047640562057495, "logits/rejected": -2.9399123191833496, "logps/chosen": -273.14666748046875, "logps/rejected": -232.124267578125, "loss": 0.6811, "positive_losses": 0.0, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.05829363316297531, "rewards/margins": 0.019574137404561043, "rewards/margins_max": 0.040946681052446365, "rewards/margins_min": -0.001798404031433165, "rewards/margins_std": 0.03022533655166626, "rewards/rejected": 0.03871949762105942, "step": 450 }, { "dpo_losses": 0.6802719831466675, "epoch": 1.4, "grad_norm": 2.1783384024286963, "learning_rate": 3.2235954771804384e-07, "logits/chosen": -2.899862051010132, "logits/rejected": -2.8675050735473633, "logps/chosen": -288.083740234375, "logps/rejected": -268.29876708984375, "loss": 0.683, "positive_losses": 0.0, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 0.0625755786895752, "rewards/margins": 0.026135718449950218, "rewards/margins_max": 0.04849334806203842, "rewards/margins_min": 0.0037780862767249346, "rewards/margins_std": 0.03161846846342087, "rewards/rejected": 0.03643985837697983, "step": 460 }, { "dpo_losses": 0.6933653354644775, "epoch": 1.43, "grad_norm": 6.179682870277185, "learning_rate": 3.138500378916752e-07, "logits/chosen": -2.8621251583099365, "logits/rejected": -2.8821539878845215, "logps/chosen": -181.8573455810547, "logps/rejected": -222.8794708251953, "loss": 0.6844, "positive_losses": 0.0033210753463208675, "rewards/accuracies": 0.44999998807907104, "rewards/chosen": 0.04994913935661316, "rewards/margins": -5.03659236983367e-07, "rewards/margins_max": 0.01866455376148224, "rewards/margins_min": -0.018665555864572525, "rewards/margins_std": 0.0263963732868433, "rewards/rejected": 0.04994964599609375, "step": 470 }, { "dpo_losses": 0.6902952790260315, "epoch": 1.46, "grad_norm": 2.096436783641514, "learning_rate": 3.0526062017313247e-07, "logits/chosen": -2.8722407817840576, "logits/rejected": -2.9027137756347656, "logps/chosen": -245.9871826171875, "logps/rejected": -332.47283935546875, "loss": 0.6832, "positive_losses": 0.0, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": 0.0516946017742157, "rewards/margins": 0.006326904986053705, "rewards/margins_max": 0.03602270036935806, "rewards/margins_min": -0.023368891328573227, "rewards/margins_std": 0.041996195912361145, "rewards/rejected": 0.04536769539117813, "step": 480 }, { "dpo_losses": 0.6820996999740601, "epoch": 1.49, "grad_norm": 2.214654285326634, "learning_rate": 2.966020441608114e-07, "logits/chosen": -2.940601110458374, "logits/rejected": -2.950160026550293, "logps/chosen": -408.1592712402344, "logps/rejected": -367.73052978515625, "loss": 0.683, "positive_losses": 0.10776062309741974, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.06694754213094711, "rewards/margins": 0.02292126789689064, "rewards/margins_max": 0.04947842285037041, "rewards/margins_min": -0.0036358858924359083, "rewards/margins_std": 0.037557490170001984, "rewards/rejected": 0.04402626305818558, "step": 490 }, { "dpo_losses": 0.6780664324760437, "epoch": 1.52, "grad_norm": 2.3489902153022464, "learning_rate": 2.8788514600424653e-07, "logits/chosen": -2.9777138233184814, "logits/rejected": -2.8717598915100098, "logps/chosen": -271.792236328125, "logps/rejected": -217.39794921875, "loss": 0.6776, "positive_losses": 0.04692535474896431, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.07212454080581665, "rewards/margins": 0.031092515215277672, "rewards/margins_max": 0.05294063687324524, "rewards/margins_min": 0.009244391694664955, "rewards/margins_std": 0.03089791163802147, "rewards/rejected": 0.04103202372789383, "step": 500 }, { "epoch": 1.52, "eval_dpo_losses": 0.6824542880058289, "eval_logits/chosen": -2.8494935035705566, "eval_logits/rejected": -2.8047306537628174, "eval_logps/chosen": -278.7507629394531, "eval_logps/rejected": -254.91441345214844, "eval_loss": 0.6883726716041565, "eval_positive_losses": 0.04435727000236511, "eval_rewards/accuracies": 0.6904761791229248, "eval_rewards/chosen": 0.06470431387424469, "eval_rewards/margins": 0.02202197164297104, "eval_rewards/margins_max": 0.07098579406738281, "eval_rewards/margins_min": -0.025560587644577026, "eval_rewards/margins_std": 0.04328176751732826, "eval_rewards/rejected": 0.042682331055402756, "eval_runtime": 282.2305, "eval_samples_per_second": 7.086, "eval_steps_per_second": 0.223, "step": 500 }, { "dpo_losses": 0.684786319732666, "epoch": 1.55, "grad_norm": 1.8216734943221224, "learning_rate": 2.791208348427426e-07, "logits/chosen": -2.806025981903076, "logits/rejected": -2.7332119941711426, "logps/chosen": -350.2364501953125, "logps/rejected": -264.30572509765625, "loss": 0.6831, "positive_losses": 0.04189910739660263, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.05516824126243591, "rewards/margins": 0.017182758077979088, "rewards/margins_max": 0.042931947857141495, "rewards/margins_min": -0.008566427044570446, "rewards/margins_std": 0.036414846777915955, "rewards/rejected": 0.037985485047101974, "step": 510 }, { "dpo_losses": 0.6818202137947083, "epoch": 1.58, "grad_norm": 7.65808980206578, "learning_rate": 2.703200791526611e-07, "logits/chosen": -3.0046448707580566, "logits/rejected": -2.982851505279541, "logps/chosen": -296.64495849609375, "logps/rejected": -249.4301300048828, "loss": 0.6837, "positive_losses": 0.0, "rewards/accuracies": 0.550000011920929, "rewards/chosen": 0.07770957797765732, "rewards/margins": 0.02354852482676506, "rewards/margins_max": 0.04674399644136429, "rewards/margins_min": 0.000353056937456131, "rewards/margins_std": 0.03280334919691086, "rewards/rejected": 0.05416105315089226, "step": 520 }, { "dpo_losses": 0.6779841184616089, "epoch": 1.61, "grad_norm": 2.0053415493661566, "learning_rate": 2.6149389302044487e-07, "logits/chosen": -2.849656105041504, "logits/rejected": -2.743305206298828, "logps/chosen": -389.1894836425781, "logps/rejected": -333.93048095703125, "loss": 0.6807, "positive_losses": 0.0, "rewards/accuracies": 0.800000011920929, "rewards/chosen": 0.08877243846654892, "rewards/margins": 0.031258475035429, "rewards/margins_max": 0.05623454600572586, "rewards/margins_min": 0.006282416172325611, "rewards/margins_std": 0.03532148897647858, "rewards/rejected": 0.05751396343111992, "step": 530 }, { "dpo_losses": 0.6694897413253784, "epoch": 1.64, "grad_norm": 1.8763346054855339, "learning_rate": 2.526533223585641e-07, "logits/chosen": -2.7003915309906006, "logits/rejected": -2.768725872039795, "logps/chosen": -374.7425231933594, "logps/rejected": -326.8890075683594, "loss": 0.6778, "positive_losses": 0.0, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 0.08555268496274948, "rewards/margins": 0.048603277653455734, "rewards/margins_max": 0.07389990985393524, "rewards/margins_min": 0.023306647315621376, "rewards/margins_std": 0.0357748381793499, "rewards/rejected": 0.03694940358400345, "step": 540 }, { "dpo_losses": 0.6740890741348267, "epoch": 1.67, "grad_norm": 7.77121120757971, "learning_rate": 2.438094310816307e-07, "logits/chosen": -2.7915353775024414, "logits/rejected": -2.8357605934143066, "logps/chosen": -232.6016387939453, "logps/rejected": -294.88299560546875, "loss": 0.6826, "positive_losses": 0.0, "rewards/accuracies": 0.75, "rewards/chosen": 0.06982794404029846, "rewards/margins": 0.03929734230041504, "rewards/margins_max": 0.07744838297367096, "rewards/margins_min": 0.0011462997645139694, "rewards/margins_std": 0.05395371839404106, "rewards/rejected": 0.030530598014593124, "step": 550 }, { "dpo_losses": 0.6748778820037842, "epoch": 1.7, "grad_norm": 2.150634086609646, "learning_rate": 2.3497328725998406e-07, "logits/chosen": -2.8007655143737793, "logits/rejected": -2.8195149898529053, "logps/chosen": -272.33734130859375, "logps/rejected": -247.6636199951172, "loss": 0.6794, "positive_losses": 0.18440398573875427, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 0.08441298454999924, "rewards/margins": 0.03730101138353348, "rewards/margins_max": 0.05858795717358589, "rewards/margins_min": 0.016014060005545616, "rewards/margins_std": 0.030104290693998337, "rewards/rejected": 0.047111984342336655, "step": 560 }, { "dpo_losses": 0.6834501028060913, "epoch": 1.73, "grad_norm": 1.9329413652442144, "learning_rate": 2.261559492680755e-07, "logits/chosen": -3.00966215133667, "logits/rejected": -2.9343342781066895, "logps/chosen": -364.2731018066406, "logps/rejected": -270.2425537109375, "loss": 0.6883, "positive_losses": 0.03923187404870987, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.07380800694227219, "rewards/margins": 0.020400431007146835, "rewards/margins_max": 0.051741547882556915, "rewards/margins_min": -0.010940684005618095, "rewards/margins_std": 0.04432303458452225, "rewards/rejected": 0.05340757966041565, "step": 570 }, { "dpo_losses": 0.683830738067627, "epoch": 1.76, "grad_norm": 6.498857250997875, "learning_rate": 2.1736845194498716e-07, "logits/chosen": -2.8374483585357666, "logits/rejected": -2.778815507888794, "logps/chosen": -294.1668395996094, "logps/rejected": -277.2710876464844, "loss": 0.6838, "positive_losses": 0.0, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 0.0776522159576416, "rewards/margins": 0.018975799903273582, "rewards/margins_max": 0.03656230866909027, "rewards/margins_min": 0.0013892930001020432, "rewards/margins_std": 0.02487107738852501, "rewards/rejected": 0.05867641046643257, "step": 580 }, { "dpo_losses": 0.6692390441894531, "epoch": 1.79, "grad_norm": 14.442968348535452, "learning_rate": 2.0862179278440504e-07, "logits/chosen": -2.752941608428955, "logits/rejected": -2.595552921295166, "logps/chosen": -326.27947998046875, "logps/rejected": -363.86175537109375, "loss": 0.6826, "positive_losses": 0.0, "rewards/accuracies": 0.800000011920929, "rewards/chosen": 0.09450069069862366, "rewards/margins": 0.04905655235052109, "rewards/margins_max": 0.08510641753673553, "rewards/margins_min": 0.013006689958274364, "rewards/margins_std": 0.05098220705986023, "rewards/rejected": 0.04544413462281227, "step": 590 }, { "dpo_losses": 0.6739311218261719, "epoch": 1.82, "grad_norm": 1.8571868330601113, "learning_rate": 1.9992691817133024e-07, "logits/chosen": -2.814953327178955, "logits/rejected": -2.798966884613037, "logps/chosen": -228.36764526367188, "logps/rejected": -220.4230499267578, "loss": 0.677, "positive_losses": 0.0, "rewards/accuracies": 0.800000011920929, "rewards/chosen": 0.07805188000202179, "rewards/margins": 0.03935372084379196, "rewards/margins_max": 0.06514634191989899, "rewards/margins_min": 0.013561112806200981, "rewards/margins_std": 0.0364762619137764, "rewards/rejected": 0.03869815915822983, "step": 600 }, { "epoch": 1.82, "eval_dpo_losses": 0.6810709238052368, "eval_logits/chosen": -2.8493857383728027, "eval_logits/rejected": -2.804689407348633, "eval_logps/chosen": -277.5360412597656, "eval_logps/rejected": -253.9931640625, "eval_loss": 0.6872838139533997, "eval_positive_losses": 0.04589047655463219, "eval_rewards/accuracies": 0.682539701461792, "eval_rewards/chosen": 0.07685134559869766, "eval_rewards/margins": 0.02495652064681053, "eval_rewards/margins_max": 0.08033440262079239, "eval_rewards/margins_min": -0.02796892076730728, "eval_rewards/margins_std": 0.04838396981358528, "eval_rewards/rejected": 0.05189482495188713, "eval_runtime": 282.2305, "eval_samples_per_second": 7.086, "eval_steps_per_second": 0.223, "step": 600 }, { "dpo_losses": 0.6791930794715881, "epoch": 1.85, "grad_norm": 2.108916508331074, "learning_rate": 1.912947096827513e-07, "logits/chosen": -2.977818250656128, "logits/rejected": -2.8510375022888184, "logps/chosen": -303.4037170410156, "logps/rejected": -240.5347137451172, "loss": 0.6825, "positive_losses": 0.0, "rewards/accuracies": 0.75, "rewards/chosen": 0.08365954458713531, "rewards/margins": 0.028567474335432053, "rewards/margins_max": 0.05369936302304268, "rewards/margins_min": 0.003435590071603656, "rewards/margins_std": 0.03554185479879379, "rewards/rejected": 0.05509205535054207, "step": 610 }, { "dpo_losses": 0.6858159303665161, "epoch": 1.88, "grad_norm": 2.0427241279215775, "learning_rate": 1.8273597046942452e-07, "logits/chosen": -2.8828835487365723, "logits/rejected": -2.7817318439483643, "logps/chosen": -252.5178985595703, "logps/rejected": -227.63070678710938, "loss": 0.6838, "positive_losses": 0.16164550185203552, "rewards/accuracies": 0.550000011920929, "rewards/chosen": 0.06277048587799072, "rewards/margins": 0.015206271782517433, "rewards/margins_max": 0.04130411148071289, "rewards/margins_min": -0.010891567915678024, "rewards/margins_std": 0.03690791875123978, "rewards/rejected": 0.04756421223282814, "step": 620 }, { "dpo_losses": 0.6795656681060791, "epoch": 1.91, "grad_norm": 4.79445166991863, "learning_rate": 1.742614117358029e-07, "logits/chosen": -2.863675594329834, "logits/rejected": -2.8280770778656006, "logps/chosen": -238.7898712158203, "logps/rejected": -213.51846313476562, "loss": 0.6827, "positive_losses": 0.0, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.06867232173681259, "rewards/margins": 0.028171828016638756, "rewards/margins_max": 0.06020760536193848, "rewards/margins_min": -0.0038639488629996777, "rewards/margins_std": 0.04530543461441994, "rewards/rejected": 0.040500491857528687, "step": 630 }, { "dpo_losses": 0.6790474057197571, "epoch": 1.95, "grad_norm": 1.9164560727509983, "learning_rate": 1.6588163933503686e-07, "logits/chosen": -2.7663025856018066, "logits/rejected": -2.7693681716918945, "logps/chosen": -180.65650939941406, "logps/rejected": -231.96517944335938, "loss": 0.6845, "positive_losses": 0.0, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.05268232896924019, "rewards/margins": 0.028681308031082153, "rewards/margins_max": 0.053858645260334015, "rewards/margins_min": 0.0035039703361690044, "rewards/margins_std": 0.035606130957603455, "rewards/rejected": 0.02400101348757744, "step": 640 }, { "dpo_losses": 0.682746946811676, "epoch": 1.98, "grad_norm": 2.1910688443112587, "learning_rate": 1.5760714049581947e-07, "logits/chosen": -2.835636615753174, "logits/rejected": -2.823740243911743, "logps/chosen": -217.59878540039062, "logps/rejected": -202.2057342529297, "loss": 0.6825, "positive_losses": 0.016841506585478783, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.05880022048950195, "rewards/margins": 0.021725308150053024, "rewards/margins_max": 0.05336114764213562, "rewards/margins_min": -0.009910527616739273, "rewards/margins_std": 0.04473983123898506, "rewards/rejected": 0.03707490488886833, "step": 650 }, { "dpo_losses": 0.6814676523208618, "epoch": 2.01, "grad_norm": 1.922230509309539, "learning_rate": 1.4944827069769122e-07, "logits/chosen": -3.023953914642334, "logits/rejected": -2.9771876335144043, "logps/chosen": -332.7305603027344, "logps/rejected": -338.9562072753906, "loss": 0.6765, "positive_losses": 0.052425384521484375, "rewards/accuracies": 0.75, "rewards/chosen": 0.054904378950595856, "rewards/margins": 0.02420732006430626, "rewards/margins_max": 0.05272083729505539, "rewards/margins_min": -0.004306197632104158, "rewards/margins_std": 0.04032420367002487, "rewards/rejected": 0.03069705329835415, "step": 660 }, { "dpo_losses": 0.6808332204818726, "epoch": 2.04, "grad_norm": 2.128169444960246, "learning_rate": 1.4141524071122656e-07, "logits/chosen": -2.90728497505188, "logits/rejected": -2.9327125549316406, "logps/chosen": -294.28411865234375, "logps/rejected": -306.5546569824219, "loss": 0.6781, "positive_losses": 0.0, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.08569607883691788, "rewards/margins": 0.026000851765275, "rewards/margins_max": 0.06097273901104927, "rewards/margins_min": -0.008971036411821842, "rewards/margins_std": 0.049457717686891556, "rewards/rejected": 0.059695225208997726, "step": 670 }, { "dpo_losses": 0.6652864217758179, "epoch": 2.07, "grad_norm": 2.0596328226644616, "learning_rate": 1.3351810381932387e-07, "logits/chosen": -2.879633665084839, "logits/rejected": -2.866821765899658, "logps/chosen": -277.6556091308594, "logps/rejected": -225.43563842773438, "loss": 0.6728, "positive_losses": 0.06625213474035263, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 0.09781384468078613, "rewards/margins": 0.05752427130937576, "rewards/margins_max": 0.09321505576372147, "rewards/margins_min": 0.02183348871767521, "rewards/margins_std": 0.0504743866622448, "rewards/rejected": 0.04028957709670067, "step": 680 }, { "dpo_losses": 0.6790655255317688, "epoch": 2.1, "grad_norm": 2.366475517541238, "learning_rate": 1.2576674323558928e-07, "logits/chosen": -2.954852819442749, "logits/rejected": -2.885873794555664, "logps/chosen": -260.78973388671875, "logps/rejected": -258.0441589355469, "loss": 0.6782, "positive_losses": 0.0, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.08070608973503113, "rewards/margins": 0.02996811270713806, "rewards/margins_max": 0.07714684307575226, "rewards/margins_min": -0.017210613936185837, "rewards/margins_std": 0.06672079861164093, "rewards/rejected": 0.050737977027893066, "step": 690 }, { "dpo_losses": 0.677202582359314, "epoch": 2.13, "grad_norm": 2.333653791310998, "learning_rate": 1.1817085973556152e-07, "logits/chosen": -2.869096279144287, "logits/rejected": -2.842957019805908, "logps/chosen": -240.80905151367188, "logps/rejected": -281.9169006347656, "loss": 0.6796, "positive_losses": 0.0, "rewards/accuracies": 0.800000011920929, "rewards/chosen": 0.08999605476856232, "rewards/margins": 0.032623276114463806, "rewards/margins_max": 0.05816454440355301, "rewards/margins_min": 0.007082013878971338, "rewards/margins_std": 0.03612080588936806, "rewards/rejected": 0.057372767478227615, "step": 700 }, { "epoch": 2.13, "eval_dpo_losses": 0.6800086498260498, "eval_logits/chosen": -2.8477256298065186, "eval_logits/rejected": -2.802640676498413, "eval_logps/chosen": -277.2366027832031, "eval_logps/rejected": -253.92018127441406, "eval_loss": 0.687153697013855, "eval_positive_losses": 0.05479403957724571, "eval_rewards/accuracies": 0.682539701461792, "eval_rewards/chosen": 0.0798461064696312, "eval_rewards/margins": 0.0272214412689209, "eval_rewards/margins_max": 0.08652784675359726, "eval_rewards/margins_min": -0.029835335910320282, "eval_rewards/margins_std": 0.05211076885461807, "eval_rewards/rejected": 0.0526246652007103, "eval_runtime": 281.0737, "eval_samples_per_second": 7.116, "eval_steps_per_second": 0.224, "step": 700 }, { "dpo_losses": 0.6692663431167603, "epoch": 2.16, "grad_norm": 1.7167790990262888, "learning_rate": 1.1073995951625664e-07, "logits/chosen": -2.9118266105651855, "logits/rejected": -2.8686835765838623, "logps/chosen": -400.4477844238281, "logps/rejected": -264.7125549316406, "loss": 0.677, "positive_losses": 0.0, "rewards/accuracies": 0.75, "rewards/chosen": 0.10832663625478745, "rewards/margins": 0.0492747537791729, "rewards/margins_max": 0.08400574326515198, "rewards/margins_min": 0.014543762430548668, "rewards/margins_std": 0.049117036163806915, "rewards/rejected": 0.05905189365148544, "step": 710 }, { "dpo_losses": 0.670403778553009, "epoch": 2.19, "grad_norm": 1.8644942647733735, "learning_rate": 1.0348334229922676e-07, "logits/chosen": -2.9610977172851562, "logits/rejected": -2.86533784866333, "logps/chosen": -368.39862060546875, "logps/rejected": -269.9642028808594, "loss": 0.6754, "positive_losses": 0.0, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.08919496089220047, "rewards/margins": 0.04688384756445885, "rewards/margins_max": 0.08247358351945877, "rewards/margins_min": 0.01129411906003952, "rewards/margins_std": 0.0503314845263958, "rewards/rejected": 0.04231110215187073, "step": 720 }, { "dpo_losses": 0.6769901514053345, "epoch": 2.22, "grad_norm": 2.180859431848076, "learning_rate": 9.6410089692021e-08, "logits/chosen": -2.9250197410583496, "logits/rejected": -2.9387307167053223, "logps/chosen": -361.2002868652344, "logps/rejected": -294.02313232421875, "loss": 0.6754, "positive_losses": 0.214344784617424, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.07332402467727661, "rewards/margins": 0.033169932663440704, "rewards/margins_max": 0.05973232537508011, "rewards/margins_min": 0.0066075399518013, "rewards/margins_std": 0.037564896047115326, "rewards/rejected": 0.04015409201383591, "step": 730 }, { "dpo_losses": 0.6715595126152039, "epoch": 2.25, "grad_norm": 2.04722557268885, "learning_rate": 8.95290538226148e-08, "logits/chosen": -2.7215945720672607, "logits/rejected": -2.681288957595825, "logps/chosen": -238.29833984375, "logps/rejected": -157.76754760742188, "loss": 0.6755, "positive_losses": 0.0, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.08287638425827026, "rewards/margins": 0.04476654529571533, "rewards/margins_max": 0.07331255823373795, "rewards/margins_min": 0.016220541670918465, "rewards/margins_std": 0.040370143949985504, "rewards/rejected": 0.03810984641313553, "step": 740 }, { "dpo_losses": 0.6766847372055054, "epoch": 2.28, "grad_norm": 7.254140526558107, "learning_rate": 8.284884626103164e-08, "logits/chosen": -2.8504419326782227, "logits/rejected": -2.817267656326294, "logps/chosen": -230.2511749267578, "logps/rejected": -262.9688415527344, "loss": 0.6787, "positive_losses": 0.0, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 0.07979516685009003, "rewards/margins": 0.03387243300676346, "rewards/margins_max": 0.05933348461985588, "rewards/margins_min": 0.008411377668380737, "rewards/margins_std": 0.036007367074489594, "rewards/rejected": 0.04592275246977806, "step": 750 }, { "dpo_losses": 0.6724633574485779, "epoch": 2.31, "grad_norm": 2.1195303141123993, "learning_rate": 7.637782724202072e-08, "logits/chosen": -2.8214287757873535, "logits/rejected": -2.8338170051574707, "logps/chosen": -247.1202850341797, "logps/rejected": -217.5040740966797, "loss": 0.684, "positive_losses": 0.0, "rewards/accuracies": 0.8999999761581421, "rewards/chosen": 0.07794385403394699, "rewards/margins": 0.04256601259112358, "rewards/margins_max": 0.07706017792224884, "rewards/margins_min": 0.008071859367191792, "rewards/margins_std": 0.0487821027636528, "rewards/rejected": 0.03537784516811371, "step": 760 }, { "dpo_losses": 0.6884227991104126, "epoch": 2.34, "grad_norm": 6.591694443648593, "learning_rate": 7.01240952022801e-08, "logits/chosen": -2.8677477836608887, "logits/rejected": -2.832568883895874, "logps/chosen": -238.2762451171875, "logps/rejected": -235.57272338867188, "loss": 0.6773, "positive_losses": 0.13628844916820526, "rewards/accuracies": 0.550000011920929, "rewards/chosen": 0.06362812221050262, "rewards/margins": 0.009944443590939045, "rewards/margins_max": 0.03966521471738815, "rewards/margins_min": -0.019776327535510063, "rewards/margins_std": 0.04203151911497116, "rewards/rejected": 0.053683679550886154, "step": 770 }, { "dpo_losses": 0.6836405396461487, "epoch": 2.37, "grad_norm": 2.3289966359685415, "learning_rate": 6.409547664531733e-08, "logits/chosen": -2.867485523223877, "logits/rejected": -2.867396831512451, "logps/chosen": -221.3419189453125, "logps/rejected": -208.03866577148438, "loss": 0.6768, "positive_losses": 0.0, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": 0.06145999953150749, "rewards/margins": 0.019474489614367485, "rewards/margins_max": 0.04479853808879852, "rewards/margins_min": -0.00584955420345068, "rewards/margins_std": 0.03581360727548599, "rewards/rejected": 0.04198550805449486, "step": 780 }, { "dpo_losses": 0.6730223894119263, "epoch": 2.4, "grad_norm": 2.003874601891291, "learning_rate": 5.829951634663324e-08, "logits/chosen": -2.7790565490722656, "logits/rejected": -2.7391462326049805, "logps/chosen": -245.35067749023438, "logps/rejected": -180.18910217285156, "loss": 0.6751, "positive_losses": 0.0, "rewards/accuracies": 0.800000011920929, "rewards/chosen": 0.08253587782382965, "rewards/margins": 0.04125886410474777, "rewards/margins_max": 0.0634620413184166, "rewards/margins_min": 0.019055673852562904, "rewards/margins_std": 0.03140005096793175, "rewards/rejected": 0.041277021169662476, "step": 790 }, { "dpo_losses": 0.6658504605293274, "epoch": 2.43, "grad_norm": 2.0012405839420944, "learning_rate": 5.274346791148601e-08, "logits/chosen": -2.9153144359588623, "logits/rejected": -2.84319806098938, "logps/chosen": -366.9844665527344, "logps/rejected": -288.7860412597656, "loss": 0.6778, "positive_losses": 0.0, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 0.08800138533115387, "rewards/margins": 0.05663619562983513, "rewards/margins_max": 0.09046684950590134, "rewards/margins_min": 0.022805536165833473, "rewards/margins_std": 0.04784378036856651, "rewards/rejected": 0.031365178525447845, "step": 800 }, { "epoch": 2.43, "eval_dpo_losses": 0.6795361638069153, "eval_logits/chosen": -2.8473622798919678, "eval_logits/rejected": -2.8024210929870605, "eval_logps/chosen": -277.22216796875, "eval_logps/rejected": -254.00743103027344, "eval_loss": 0.6875322461128235, "eval_positive_losses": 0.060370072722435, "eval_rewards/accuracies": 0.682539701461792, "eval_rewards/chosen": 0.07999046891927719, "eval_rewards/margins": 0.0282380823045969, "eval_rewards/margins_max": 0.08973579108715057, "eval_rewards/margins_min": -0.030655015259981155, "eval_rewards/margins_std": 0.05399897322058678, "eval_rewards/rejected": 0.05175239220261574, "eval_runtime": 281.3076, "eval_samples_per_second": 7.11, "eval_steps_per_second": 0.224, "step": 800 }, { "dpo_losses": 0.6727338433265686, "epoch": 2.46, "grad_norm": 7.411310583604656, "learning_rate": 4.743428469705335e-08, "logits/chosen": -2.6556921005249023, "logits/rejected": -2.691716432571411, "logps/chosen": -270.0729064941406, "logps/rejected": -222.0420379638672, "loss": 0.6752, "positive_losses": 0.0, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.08230028301477432, "rewards/margins": 0.04306231066584587, "rewards/margins_max": 0.09098808467388153, "rewards/margins_min": -0.004863465670496225, "rewards/margins_std": 0.06777729094028473, "rewards/rejected": 0.03923797979950905, "step": 810 }, { "dpo_losses": 0.6766294240951538, "epoch": 2.49, "grad_norm": 2.0160351712928146, "learning_rate": 4.2378611110352707e-08, "logits/chosen": -2.8572134971618652, "logits/rejected": -2.9511802196502686, "logps/chosen": -193.8530731201172, "logps/rejected": -267.66522216796875, "loss": 0.6747, "positive_losses": 0.0, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.06872855871915817, "rewards/margins": 0.03442535921931267, "rewards/margins_max": 0.07178723812103271, "rewards/margins_min": -0.0029365241061896086, "rewards/margins_std": 0.05283767729997635, "rewards/rejected": 0.034303199499845505, "step": 820 }, { "dpo_losses": 0.6828433275222778, "epoch": 2.52, "grad_norm": 2.1498185591320773, "learning_rate": 3.75827742928107e-08, "logits/chosen": -2.9401209354400635, "logits/rejected": -2.8616251945495605, "logps/chosen": -311.86260986328125, "logps/rejected": -264.8041076660156, "loss": 0.6773, "positive_losses": 0.0, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.087630495429039, "rewards/margins": 0.022305091843008995, "rewards/margins_max": 0.07450206577777863, "rewards/margins_min": -0.029891883954405785, "rewards/margins_std": 0.0738176628947258, "rewards/rejected": 0.06532540172338486, "step": 830 }, { "dpo_losses": 0.6704001426696777, "epoch": 2.55, "grad_norm": 11.453340827604897, "learning_rate": 3.305277620188826e-08, "logits/chosen": -2.805208921432495, "logits/rejected": -2.840792179107666, "logps/chosen": -336.3541564941406, "logps/rejected": -344.42413330078125, "loss": 0.6733, "positive_losses": 0.0, "rewards/accuracies": 0.800000011920929, "rewards/chosen": 0.09611042588949203, "rewards/margins": 0.04694870114326477, "rewards/margins_max": 0.08145497739315033, "rewards/margins_min": 0.012442430481314659, "rewards/margins_std": 0.04879923164844513, "rewards/rejected": 0.04916173219680786, "step": 840 }, { "dpo_losses": 0.6803882122039795, "epoch": 2.58, "grad_norm": 2.3437007644568086, "learning_rate": 2.879428609967105e-08, "logits/chosen": -2.921419620513916, "logits/rejected": -2.8526148796081543, "logps/chosen": -216.5243377685547, "logps/rejected": -294.4681091308594, "loss": 0.6739, "positive_losses": 0.0, "rewards/accuracies": 0.800000011920929, "rewards/chosen": 0.07691861689090729, "rewards/margins": 0.02676830254495144, "rewards/margins_max": 0.05155067518353462, "rewards/margins_min": 0.001985931070521474, "rewards/margins_std": 0.035047560930252075, "rewards/rejected": 0.0501503124833107, "step": 850 }, { "dpo_losses": 0.679145872592926, "epoch": 2.61, "grad_norm": 2.249308373857521, "learning_rate": 2.4812633457826666e-08, "logits/chosen": -2.868685007095337, "logits/rejected": -2.853390693664551, "logps/chosen": -259.72357177734375, "logps/rejected": -335.20465087890625, "loss": 0.6762, "positive_losses": 0.0, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 0.0714501366019249, "rewards/margins": 0.028765708208084106, "rewards/margins_max": 0.051595449447631836, "rewards/margins_min": 0.00593597162514925, "rewards/margins_std": 0.03228612244129181, "rewards/rejected": 0.04268442839384079, "step": 860 }, { "dpo_losses": 0.6712166666984558, "epoch": 2.64, "grad_norm": 1.888865132587614, "learning_rate": 2.1112801287806375e-08, "logits/chosen": -2.803732395172119, "logits/rejected": -2.8339738845825195, "logps/chosen": -336.16632080078125, "logps/rejected": -346.7503662109375, "loss": 0.6767, "positive_losses": 0.0, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.10950883477926254, "rewards/margins": 0.04584169760346413, "rewards/margins_max": 0.0953935980796814, "rewards/margins_min": -0.003710202407091856, "rewards/margins_std": 0.07007695734500885, "rewards/rejected": 0.06366713345050812, "step": 870 }, { "dpo_losses": 0.663690447807312, "epoch": 2.67, "grad_norm": 2.5335259272331125, "learning_rate": 1.7699419904640022e-08, "logits/chosen": -2.864894151687622, "logits/rejected": -2.822705030441284, "logps/chosen": -253.73703002929688, "logps/rejected": -260.36767578125, "loss": 0.6763, "positive_losses": 0.0, "rewards/accuracies": 0.800000011920929, "rewards/chosen": 0.1016063317656517, "rewards/margins": 0.060955654829740524, "rewards/margins_max": 0.10850463807582855, "rewards/margins_min": 0.013406666927039623, "rewards/margins_std": 0.06724441796541214, "rewards/rejected": 0.040650688111782074, "step": 880 }, { "dpo_losses": 0.6715737581253052, "epoch": 2.71, "grad_norm": 11.524954878911132, "learning_rate": 1.4576761132127912e-08, "logits/chosen": -3.0019936561584473, "logits/rejected": -2.8649494647979736, "logps/chosen": -330.13470458984375, "logps/rejected": -248.55508422851562, "loss": 0.674, "positive_losses": 0.0, "rewards/accuracies": 0.800000011920929, "rewards/chosen": 0.07323987782001495, "rewards/margins": 0.044342849403619766, "rewards/margins_max": 0.0754455104470253, "rewards/margins_min": 0.013240176253020763, "rewards/margins_std": 0.04398581385612488, "rewards/rejected": 0.028897028416395187, "step": 890 }, { "dpo_losses": 0.6749640107154846, "epoch": 2.74, "grad_norm": 1.8036243560449348, "learning_rate": 1.1748732956682023e-08, "logits/chosen": -2.7051098346710205, "logits/rejected": -2.655970335006714, "logps/chosen": -257.9394836425781, "logps/rejected": -181.5701446533203, "loss": 0.6739, "positive_losses": 0.0, "rewards/accuracies": 0.6499999761581421, "rewards/chosen": 0.07508902996778488, "rewards/margins": 0.03800603747367859, "rewards/margins_max": 0.09310616552829742, "rewards/margins_min": -0.017094094306230545, "rewards/margins_std": 0.07792335003614426, "rewards/rejected": 0.03708299994468689, "step": 900 }, { "epoch": 2.74, "eval_dpo_losses": 0.679327130317688, "eval_logits/chosen": -2.8477160930633545, "eval_logits/rejected": -2.802847385406494, "eval_logps/chosen": -277.20367431640625, "eval_logps/rejected": -254.03451538085938, "eval_loss": 0.6877949833869934, "eval_positive_losses": 0.06513993442058563, "eval_rewards/accuracies": 0.670634925365448, "eval_rewards/chosen": 0.08017554879188538, "eval_rewards/margins": 0.028694164007902145, "eval_rewards/margins_max": 0.09142320603132248, "eval_rewards/margins_min": -0.03173976391553879, "eval_rewards/margins_std": 0.05502929911017418, "eval_rewards/rejected": 0.05148139223456383, "eval_runtime": 282.246, "eval_samples_per_second": 7.086, "eval_steps_per_second": 0.223, "step": 900 }, { "dpo_losses": 0.6807221174240112, "epoch": 2.77, "grad_norm": 7.587296620491664, "learning_rate": 9.218874636507556e-09, "logits/chosen": -2.8844008445739746, "logits/rejected": -2.8472609519958496, "logps/chosen": -232.85934448242188, "logps/rejected": -256.0882873535156, "loss": 0.6782, "positive_losses": 0.17994289100170135, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.07232363522052765, "rewards/margins": 0.02574983797967434, "rewards/margins_max": 0.06079038232564926, "rewards/margins_min": -0.009290714748203754, "rewards/margins_std": 0.049554821103811264, "rewards/rejected": 0.04657380282878876, "step": 910 }, { "dpo_losses": 0.674708902835846, "epoch": 2.8, "grad_norm": 2.0209217117752267, "learning_rate": 6.9903522722450645e-09, "logits/chosen": -2.6785788536071777, "logits/rejected": -2.683105945587158, "logps/chosen": -205.6979522705078, "logps/rejected": -194.4645233154297, "loss": 0.6772, "positive_losses": 0.0, "rewards/accuracies": 0.800000011920929, "rewards/chosen": 0.06881477683782578, "rewards/margins": 0.03808169811964035, "rewards/margins_max": 0.06708801537752151, "rewards/margins_min": 0.00907537154853344, "rewards/margins_std": 0.041021138429641724, "rewards/rejected": 0.030733084306120872, "step": 920 }, { "dpo_losses": 0.6782993078231812, "epoch": 2.83, "grad_norm": 2.1553504283654608, "learning_rate": 5.065954844616721e-09, "logits/chosen": -2.8473258018493652, "logits/rejected": -2.778991222381592, "logps/chosen": -272.9423522949219, "logps/rejected": -323.91290283203125, "loss": 0.6829, "positive_losses": 0.127574160695076, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.07656483352184296, "rewards/margins": 0.03158252686262131, "rewards/margins_max": 0.08533596247434616, "rewards/margins_min": -0.022170905023813248, "rewards/margins_std": 0.07601883262395859, "rewards/rejected": 0.04498231038451195, "step": 930 }, { "dpo_losses": 0.6669495701789856, "epoch": 2.86, "grad_norm": 1.8894258589844486, "learning_rate": 3.4480907240357904e-09, "logits/chosen": -2.886190176010132, "logits/rejected": -2.852233409881592, "logps/chosen": -237.05038452148438, "logps/rejected": -192.5042724609375, "loss": 0.6785, "positive_losses": 0.08552627265453339, "rewards/accuracies": 0.800000011920929, "rewards/chosen": 0.08725001662969589, "rewards/margins": 0.05445738509297371, "rewards/margins_max": 0.08135871589183807, "rewards/margins_min": 0.027556050568819046, "rewards/margins_std": 0.03804423287510872, "rewards/rejected": 0.03279264643788338, "step": 940 }, { "dpo_losses": 0.6740903854370117, "epoch": 2.89, "grad_norm": 7.272085291531218, "learning_rate": 2.1387846565474044e-09, "logits/chosen": -2.7927393913269043, "logits/rejected": -2.80344295501709, "logps/chosen": -295.1185607910156, "logps/rejected": -264.1958312988281, "loss": 0.6773, "positive_losses": 0.0, "rewards/accuracies": 0.8500000238418579, "rewards/chosen": 0.09159694612026215, "rewards/margins": 0.039398252964019775, "rewards/margins_max": 0.07913588732481003, "rewards/margins_min": -0.00033939722925424576, "rewards/margins_std": 0.056197524070739746, "rewards/rejected": 0.05219869688153267, "step": 950 }, { "dpo_losses": 0.676175594329834, "epoch": 2.92, "grad_norm": 2.0704055477321535, "learning_rate": 1.1396752298723499e-09, "logits/chosen": -2.6965384483337402, "logits/rejected": -2.7728757858276367, "logps/chosen": -243.5664825439453, "logps/rejected": -246.42904663085938, "loss": 0.6762, "positive_losses": 0.0, "rewards/accuracies": 0.699999988079071, "rewards/chosen": 0.07029319554567337, "rewards/margins": 0.03469633311033249, "rewards/margins_max": 0.0634746104478836, "rewards/margins_min": 0.005918048322200775, "rewards/margins_std": 0.040698640048503876, "rewards/rejected": 0.03559685871005058, "step": 960 }, { "dpo_losses": 0.6741344332695007, "epoch": 2.95, "grad_norm": 4.375655860786601, "learning_rate": 4.520128227257225e-10, "logits/chosen": -2.9053943157196045, "logits/rejected": -2.903526782989502, "logps/chosen": -324.132568359375, "logps/rejected": -269.60394287109375, "loss": 0.6787, "positive_losses": 0.0, "rewards/accuracies": 0.6000000238418579, "rewards/chosen": 0.10518592596054077, "rewards/margins": 0.04040554165840149, "rewards/margins_max": 0.09129390865564346, "rewards/margins_min": -0.010482819750905037, "rewards/margins_std": 0.07196702063083649, "rewards/rejected": 0.06478038430213928, "step": 970 }, { "dpo_losses": 0.6782342791557312, "epoch": 2.98, "grad_norm": 2.102961531885413, "learning_rate": 7.665803997633502e-11, "logits/chosen": -2.785641670227051, "logits/rejected": -2.674614667892456, "logps/chosen": -221.5739288330078, "logps/rejected": -235.2669677734375, "loss": 0.6753, "positive_losses": 0.0, "rewards/accuracies": 0.75, "rewards/chosen": 0.06868946552276611, "rewards/margins": 0.030555952340364456, "rewards/margins_max": 0.06132732704281807, "rewards/margins_min": -0.00021542570902965963, "rewards/margins_std": 0.04351729154586792, "rewards/rejected": 0.03813350945711136, "step": 980 }, { "epoch": 3.0, "step": 987, "total_flos": 0.0, "train_loss": 0.684076373458754, "train_runtime": 8695.656, "train_samples_per_second": 1.816, "train_steps_per_second": 0.114 } ], "logging_steps": 10, "max_steps": 987, "num_input_tokens_seen": 0, "num_train_epochs": 3, "save_steps": 100, "total_flos": 0.0, "train_batch_size": 2, "trial_name": null, "trial_params": null }