| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 100, | |
| "global_step": 684, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "dpo_losses": 0.6931471824645996, | |
| "epoch": 0.0, | |
| "grad_norm": 2.025402631880394, | |
| "learning_rate": 7.246376811594204e-08, | |
| "logits/chosen": -2.961127519607544, | |
| "logits/rejected": -2.9461119174957275, | |
| "logps/chosen": -261.90582275390625, | |
| "logps/rejected": -270.03265380859375, | |
| "loss": 0.6931, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/margins_max": 0.0, | |
| "rewards/margins_min": 0.0, | |
| "rewards/margins_std": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "dpo_losses": 0.6932222843170166, | |
| "epoch": 0.01, | |
| "grad_norm": 1.892116368261662, | |
| "learning_rate": 7.246376811594204e-07, | |
| "logits/chosen": -2.875087022781372, | |
| "logits/rejected": -2.855910062789917, | |
| "logps/chosen": -217.50634765625, | |
| "logps/rejected": -222.0803985595703, | |
| "loss": 0.6974, | |
| "positive_losses": 0.04892720282077789, | |
| "rewards/accuracies": 0.4444444477558136, | |
| "rewards/chosen": 0.00041001950739882886, | |
| "rewards/margins": -0.0001489536080043763, | |
| "rewards/margins_max": 0.0012003988958895206, | |
| "rewards/margins_min": -0.0014983059372752905, | |
| "rewards/margins_std": 0.0019082725048065186, | |
| "rewards/rejected": 0.0005589731154032052, | |
| "step": 10 | |
| }, | |
| { | |
| "dpo_losses": 0.6928491592407227, | |
| "epoch": 0.03, | |
| "grad_norm": 10.889312446124245, | |
| "learning_rate": 1.4492753623188408e-06, | |
| "logits/chosen": -2.855677366256714, | |
| "logits/rejected": -2.8727664947509766, | |
| "logps/chosen": -228.65463256835938, | |
| "logps/rejected": -176.28146362304688, | |
| "loss": 0.695, | |
| "positive_losses": 0.026834487915039062, | |
| "rewards/accuracies": 0.44999998807907104, | |
| "rewards/chosen": 0.0038786642253398895, | |
| "rewards/margins": 0.0005991062498651445, | |
| "rewards/margins_max": 0.0024178135208785534, | |
| "rewards/margins_min": -0.001219600671902299, | |
| "rewards/margins_std": 0.0025720400735735893, | |
| "rewards/rejected": 0.0032795581500977278, | |
| "step": 20 | |
| }, | |
| { | |
| "dpo_losses": 0.691431999206543, | |
| "epoch": 0.04, | |
| "grad_norm": 2.0735563380689697, | |
| "learning_rate": 2.173913043478261e-06, | |
| "logits/chosen": -2.932262420654297, | |
| "logits/rejected": -2.8772940635681152, | |
| "logps/chosen": -258.99334716796875, | |
| "logps/rejected": -237.83096313476562, | |
| "loss": 0.692, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": 0.015487673692405224, | |
| "rewards/margins": 0.0034467983059585094, | |
| "rewards/margins_max": 0.008276171050965786, | |
| "rewards/margins_min": -0.001382575137540698, | |
| "rewards/margins_std": 0.006829765625298023, | |
| "rewards/rejected": 0.012040875852108002, | |
| "step": 30 | |
| }, | |
| { | |
| "dpo_losses": 0.6876205205917358, | |
| "epoch": 0.06, | |
| "grad_norm": 1.9226295416634294, | |
| "learning_rate": 2.8985507246376816e-06, | |
| "logits/chosen": -2.8300986289978027, | |
| "logits/rejected": -2.7832179069519043, | |
| "logps/chosen": -325.06231689453125, | |
| "logps/rejected": -363.68426513671875, | |
| "loss": 0.6887, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": 0.028436576947569847, | |
| "rewards/margins": 0.011200213804841042, | |
| "rewards/margins_max": 0.022198233753442764, | |
| "rewards/margins_min": 0.00020219237194396555, | |
| "rewards/margins_std": 0.015553551726043224, | |
| "rewards/rejected": 0.017236361280083656, | |
| "step": 40 | |
| }, | |
| { | |
| "dpo_losses": 0.6896201968193054, | |
| "epoch": 0.07, | |
| "grad_norm": 9.118712466857515, | |
| "learning_rate": 3.6231884057971017e-06, | |
| "logits/chosen": -2.895482301712036, | |
| "logits/rejected": -2.8222224712371826, | |
| "logps/chosen": -247.339111328125, | |
| "logps/rejected": -244.00790405273438, | |
| "loss": 0.6891, | |
| "positive_losses": 0.0010955811012536287, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": 0.03269472345709801, | |
| "rewards/margins": 0.007144673261791468, | |
| "rewards/margins_max": 0.017206599935889244, | |
| "rewards/margins_min": -0.002917253179475665, | |
| "rewards/margins_std": 0.014229713007807732, | |
| "rewards/rejected": 0.025550048798322678, | |
| "step": 50 | |
| }, | |
| { | |
| "dpo_losses": 0.678604006767273, | |
| "epoch": 0.09, | |
| "grad_norm": 1.6865302230972352, | |
| "learning_rate": 4.347826086956522e-06, | |
| "logits/chosen": -3.02363920211792, | |
| "logits/rejected": -2.9497618675231934, | |
| "logps/chosen": -302.65142822265625, | |
| "logps/rejected": -242.8329620361328, | |
| "loss": 0.6829, | |
| "positive_losses": 0.0033214569557458162, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.06167557090520859, | |
| "rewards/margins": 0.02971900999546051, | |
| "rewards/margins_max": 0.05635114759206772, | |
| "rewards/margins_min": 0.003086873795837164, | |
| "rewards/margins_std": 0.0376635305583477, | |
| "rewards/rejected": 0.03195656090974808, | |
| "step": 60 | |
| }, | |
| { | |
| "dpo_losses": 0.6720460653305054, | |
| "epoch": 0.1, | |
| "grad_norm": 7.877876119832288, | |
| "learning_rate": 4.999967381905813e-06, | |
| "logits/chosen": -3.0418922901153564, | |
| "logits/rejected": -2.9664931297302246, | |
| "logps/chosen": -266.40692138671875, | |
| "logps/rejected": -203.53610229492188, | |
| "loss": 0.6792, | |
| "positive_losses": 0.09538726508617401, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.05729568004608154, | |
| "rewards/margins": 0.04375966638326645, | |
| "rewards/margins_max": 0.05517454072833061, | |
| "rewards/margins_min": 0.032344792038202286, | |
| "rewards/margins_std": 0.0161430723965168, | |
| "rewards/rejected": 0.013536013662815094, | |
| "step": 70 | |
| }, | |
| { | |
| "dpo_losses": 0.6691193580627441, | |
| "epoch": 0.12, | |
| "grad_norm": 9.317121958881922, | |
| "learning_rate": 4.9960542403925095e-06, | |
| "logits/chosen": -2.8223726749420166, | |
| "logits/rejected": -2.7410635948181152, | |
| "logps/chosen": -249.7649383544922, | |
| "logps/rejected": -231.88986206054688, | |
| "loss": 0.6716, | |
| "positive_losses": 0.03403167799115181, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.07761463522911072, | |
| "rewards/margins": 0.05084484815597534, | |
| "rewards/margins_max": 0.10426272451877594, | |
| "rewards/margins_min": -0.0025730193592607975, | |
| "rewards/margins_std": 0.07554427534341812, | |
| "rewards/rejected": 0.026769787073135376, | |
| "step": 80 | |
| }, | |
| { | |
| "dpo_losses": 0.6618136167526245, | |
| "epoch": 0.13, | |
| "grad_norm": 4.628284010129237, | |
| "learning_rate": 4.98562917836165e-06, | |
| "logits/chosen": -2.881012201309204, | |
| "logits/rejected": -2.8446288108825684, | |
| "logps/chosen": -254.67495727539062, | |
| "logps/rejected": -195.2042694091797, | |
| "loss": 0.6694, | |
| "positive_losses": 0.05231323093175888, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.08632297068834305, | |
| "rewards/margins": 0.06475184857845306, | |
| "rewards/margins_max": 0.08643685281276703, | |
| "rewards/margins_min": 0.0430668406188488, | |
| "rewards/margins_std": 0.030667226761579514, | |
| "rewards/rejected": 0.021571118384599686, | |
| "step": 90 | |
| }, | |
| { | |
| "dpo_losses": 0.6343039274215698, | |
| "epoch": 0.15, | |
| "grad_norm": 12.853627066309556, | |
| "learning_rate": 4.968719393609757e-06, | |
| "logits/chosen": -2.973792552947998, | |
| "logits/rejected": -2.9188525676727295, | |
| "logps/chosen": -364.15399169921875, | |
| "logps/rejected": -228.38418579101562, | |
| "loss": 0.6612, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.1419464498758316, | |
| "rewards/margins": 0.12534113228321075, | |
| "rewards/margins_max": 0.19193264842033386, | |
| "rewards/margins_min": 0.05874960869550705, | |
| "rewards/margins_std": 0.09417462348937988, | |
| "rewards/rejected": 0.016605319455266, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "eval_dpo_losses": 0.6805257797241211, | |
| "eval_logits/chosen": -2.8549489974975586, | |
| "eval_logits/rejected": -2.8096561431884766, | |
| "eval_logps/chosen": -277.0838928222656, | |
| "eval_logps/rejected": -253.83172607421875, | |
| "eval_loss": 0.7167356014251709, | |
| "eval_positive_losses": 0.34227606654167175, | |
| "eval_rewards/accuracies": 0.591269850730896, | |
| "eval_rewards/chosen": 0.0813729390501976, | |
| "eval_rewards/margins": 0.027863360941410065, | |
| "eval_rewards/margins_max": 0.13411983847618103, | |
| "eval_rewards/margins_min": -0.06980105489492416, | |
| "eval_rewards/margins_std": 0.0907549038529396, | |
| "eval_rewards/rejected": 0.05350957810878754, | |
| "eval_runtime": 284.067, | |
| "eval_samples_per_second": 7.041, | |
| "eval_steps_per_second": 0.222, | |
| "step": 100 | |
| }, | |
| { | |
| "dpo_losses": 0.646482527256012, | |
| "epoch": 0.16, | |
| "grad_norm": 7.886066230281033, | |
| "learning_rate": 4.9453690018345144e-06, | |
| "logits/chosen": -2.90840482711792, | |
| "logits/rejected": -2.870948553085327, | |
| "logps/chosen": -332.84967041015625, | |
| "logps/rejected": -298.0467529296875, | |
| "loss": 0.6608, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.11125056445598602, | |
| "rewards/margins": 0.09774880856275558, | |
| "rewards/margins_max": 0.12077156454324722, | |
| "rewards/margins_min": 0.07472606003284454, | |
| "rewards/margins_std": 0.03255908936262131, | |
| "rewards/rejected": 0.01350175030529499, | |
| "step": 110 | |
| }, | |
| { | |
| "dpo_losses": 0.6566277742385864, | |
| "epoch": 0.18, | |
| "grad_norm": 1.913254174024933, | |
| "learning_rate": 4.915638921541952e-06, | |
| "logits/chosen": -2.8616955280303955, | |
| "logits/rejected": -2.8522956371307373, | |
| "logps/chosen": -277.81402587890625, | |
| "logps/rejected": -261.228759765625, | |
| "loss": 0.6473, | |
| "positive_losses": 0.04311790317296982, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.0988793820142746, | |
| "rewards/margins": 0.07747145742177963, | |
| "rewards/margins_max": 0.14120900630950928, | |
| "rewards/margins_min": 0.013733914121985435, | |
| "rewards/margins_std": 0.0901385098695755, | |
| "rewards/rejected": 0.02140791341662407, | |
| "step": 120 | |
| }, | |
| { | |
| "dpo_losses": 0.6377557516098022, | |
| "epoch": 0.19, | |
| "grad_norm": 12.904038948771282, | |
| "learning_rate": 4.879606715117019e-06, | |
| "logits/chosen": -2.95271897315979, | |
| "logits/rejected": -2.8435444831848145, | |
| "logps/chosen": -294.14813232421875, | |
| "logps/rejected": -240.53207397460938, | |
| "loss": 0.6576, | |
| "positive_losses": 0.14576569199562073, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.12051650136709213, | |
| "rewards/margins": 0.12185319513082504, | |
| "rewards/margins_max": 0.16484162211418152, | |
| "rewards/margins_min": 0.07886476814746857, | |
| "rewards/margins_std": 0.06079481169581413, | |
| "rewards/rejected": -0.0013366841012611985, | |
| "step": 130 | |
| }, | |
| { | |
| "dpo_losses": 0.640169620513916, | |
| "epoch": 0.2, | |
| "grad_norm": 16.18813154837036, | |
| "learning_rate": 4.837366386472175e-06, | |
| "logits/chosen": -3.0178513526916504, | |
| "logits/rejected": -2.9269156455993652, | |
| "logps/chosen": -279.7732238769531, | |
| "logps/rejected": -236.86874389648438, | |
| "loss": 0.6767, | |
| "positive_losses": 0.29906386137008667, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.11848002672195435, | |
| "rewards/margins": 0.11471493542194366, | |
| "rewards/margins_max": 0.2023201882839203, | |
| "rewards/margins_min": 0.02710966393351555, | |
| "rewards/margins_std": 0.12389256060123444, | |
| "rewards/rejected": 0.0037650964222848415, | |
| "step": 140 | |
| }, | |
| { | |
| "dpo_losses": 0.6285568475723267, | |
| "epoch": 0.22, | |
| "grad_norm": 6.5324825602261045, | |
| "learning_rate": 4.789028135801919e-06, | |
| "logits/chosen": -2.973548650741577, | |
| "logits/rejected": -2.9277901649475098, | |
| "logps/chosen": -281.6429443359375, | |
| "logps/rejected": -295.3494873046875, | |
| "loss": 0.8701, | |
| "positive_losses": 0.2579152584075928, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.16488775610923767, | |
| "rewards/margins": 0.14175625145435333, | |
| "rewards/margins_max": 0.25869402289390564, | |
| "rewards/margins_min": 0.02481846883893013, | |
| "rewards/margins_std": 0.1653749793767929, | |
| "rewards/rejected": 0.023131517693400383, | |
| "step": 150 | |
| }, | |
| { | |
| "dpo_losses": 0.6205035448074341, | |
| "epoch": 0.23, | |
| "grad_norm": 2.7066336555848176, | |
| "learning_rate": 4.7347180720830635e-06, | |
| "logits/chosen": -2.9675240516662598, | |
| "logits/rejected": -2.804438829421997, | |
| "logps/chosen": -316.6290283203125, | |
| "logps/rejected": -282.70538330078125, | |
| "loss": 0.6598, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.1802542507648468, | |
| "rewards/margins": 0.15980175137519836, | |
| "rewards/margins_max": 0.272554486989975, | |
| "rewards/margins_min": 0.047049008309841156, | |
| "rewards/margins_std": 0.15945644676685333, | |
| "rewards/rejected": 0.02045249193906784, | |
| "step": 160 | |
| }, | |
| { | |
| "dpo_losses": 0.6271503567695618, | |
| "epoch": 0.25, | |
| "grad_norm": 1.8801745537623247, | |
| "learning_rate": 4.674577884070811e-06, | |
| "logits/chosen": -2.929814577102661, | |
| "logits/rejected": -2.8838062286376953, | |
| "logps/chosen": -308.89947509765625, | |
| "logps/rejected": -252.41775512695312, | |
| "loss": 0.6279, | |
| "positive_losses": 0.5797370672225952, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.1764679253101349, | |
| "rewards/margins": 0.14800240099430084, | |
| "rewards/margins_max": 0.237053781747818, | |
| "rewards/margins_min": 0.058951012790203094, | |
| "rewards/margins_std": 0.12593765556812286, | |
| "rewards/rejected": 0.02846553362905979, | |
| "step": 170 | |
| }, | |
| { | |
| "dpo_losses": 0.6267830729484558, | |
| "epoch": 0.26, | |
| "grad_norm": 2.1466440934520645, | |
| "learning_rate": 4.608764470648971e-06, | |
| "logits/chosen": -2.9405906200408936, | |
| "logits/rejected": -2.8729405403137207, | |
| "logps/chosen": -290.3728942871094, | |
| "logps/rejected": -330.1247863769531, | |
| "loss": 0.6545, | |
| "positive_losses": 0.3962584435939789, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.17637397348880768, | |
| "rewards/margins": 0.14918141067028046, | |
| "rewards/margins_max": 0.28907179832458496, | |
| "rewards/margins_min": 0.00929100438952446, | |
| "rewards/margins_std": 0.19783492386341095, | |
| "rewards/rejected": 0.027192572131752968, | |
| "step": 180 | |
| }, | |
| { | |
| "dpo_losses": 0.5917172431945801, | |
| "epoch": 0.28, | |
| "grad_norm": 2.287767271676791, | |
| "learning_rate": 4.5374495314986874e-06, | |
| "logits/chosen": -2.7434186935424805, | |
| "logits/rejected": -2.7498998641967773, | |
| "logps/chosen": -312.6622619628906, | |
| "logps/rejected": -238.92544555664062, | |
| "loss": 0.6419, | |
| "positive_losses": 0.6725692749023438, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.21900752186775208, | |
| "rewards/margins": 0.22816917300224304, | |
| "rewards/margins_max": 0.34974128007888794, | |
| "rewards/margins_min": 0.10659710317850113, | |
| "rewards/margins_std": 0.17192888259887695, | |
| "rewards/rejected": -0.00916165579110384, | |
| "step": 190 | |
| }, | |
| { | |
| "dpo_losses": 0.6394025087356567, | |
| "epoch": 0.29, | |
| "grad_norm": 10.012997763234694, | |
| "learning_rate": 4.460819119153574e-06, | |
| "logits/chosen": -2.8694872856140137, | |
| "logits/rejected": -2.831519603729248, | |
| "logps/chosen": -254.72994995117188, | |
| "logps/rejected": -285.6749572753906, | |
| "loss": 0.6652, | |
| "positive_losses": 0.53265380859375, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.1096222847700119, | |
| "rewards/margins": 0.11550626903772354, | |
| "rewards/margins_max": 0.21246998012065887, | |
| "rewards/margins_min": 0.018542537465691566, | |
| "rewards/margins_std": 0.13712741434574127, | |
| "rewards/rejected": -0.005883966572582722, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "eval_dpo_losses": 0.6683889627456665, | |
| "eval_logits/chosen": -2.804701566696167, | |
| "eval_logits/rejected": -2.761190176010132, | |
| "eval_logps/chosen": -270.78045654296875, | |
| "eval_logps/rejected": -250.50535583496094, | |
| "eval_loss": 0.7424377202987671, | |
| "eval_positive_losses": 0.6157510280609131, | |
| "eval_rewards/accuracies": 0.6071428656578064, | |
| "eval_rewards/chosen": 0.14440776407718658, | |
| "eval_rewards/margins": 0.05763502046465874, | |
| "eval_rewards/margins_max": 0.23871682584285736, | |
| "eval_rewards/margins_min": -0.10859239846467972, | |
| "eval_rewards/margins_std": 0.15644660592079163, | |
| "eval_rewards/rejected": 0.08677274733781815, | |
| "eval_runtime": 283.023, | |
| "eval_samples_per_second": 7.067, | |
| "eval_steps_per_second": 0.223, | |
| "step": 200 | |
| }, | |
| { | |
| "dpo_losses": 0.6189439296722412, | |
| "epoch": 0.31, | |
| "grad_norm": 6.2964509201178664, | |
| "learning_rate": 4.379073153609896e-06, | |
| "logits/chosen": -2.980807304382324, | |
| "logits/rejected": -2.9285852909088135, | |
| "logps/chosen": -310.4910583496094, | |
| "logps/rejected": -283.2040100097656, | |
| "loss": 0.66, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.17465968430042267, | |
| "rewards/margins": 0.16720545291900635, | |
| "rewards/margins_max": 0.2678540349006653, | |
| "rewards/margins_min": 0.0665569081902504, | |
| "rewards/margins_std": 0.14233854413032532, | |
| "rewards/rejected": 0.007454232778400183, | |
| "step": 210 | |
| }, | |
| { | |
| "dpo_losses": 0.6231792569160461, | |
| "epoch": 0.32, | |
| "grad_norm": 1.9190071231894708, | |
| "learning_rate": 4.292424900758129e-06, | |
| "logits/chosen": -2.8317534923553467, | |
| "logits/rejected": -2.745687961578369, | |
| "logps/chosen": -241.54098510742188, | |
| "logps/rejected": -261.16180419921875, | |
| "loss": 0.6416, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.17161966860294342, | |
| "rewards/margins": 0.15514414012432098, | |
| "rewards/margins_max": 0.2861190140247345, | |
| "rewards/margins_min": 0.024169281125068665, | |
| "rewards/margins_std": 0.1852264106273651, | |
| "rewards/rejected": 0.016475532203912735, | |
| "step": 220 | |
| }, | |
| { | |
| "dpo_losses": 0.6433408260345459, | |
| "epoch": 0.34, | |
| "grad_norm": 2.0868151243656565, | |
| "learning_rate": 4.201100415996598e-06, | |
| "logits/chosen": -2.727468490600586, | |
| "logits/rejected": -2.6615824699401855, | |
| "logps/chosen": -234.8723602294922, | |
| "logps/rejected": -252.6638641357422, | |
| "loss": 0.6425, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.1352681964635849, | |
| "rewards/margins": 0.10586099326610565, | |
| "rewards/margins_max": 0.18672436475753784, | |
| "rewards/margins_min": 0.024997618049383163, | |
| "rewards/margins_std": 0.11435806751251221, | |
| "rewards/rejected": 0.029407206922769547, | |
| "step": 230 | |
| }, | |
| { | |
| "dpo_losses": 0.5827513933181763, | |
| "epoch": 0.35, | |
| "grad_norm": 8.16859688297942, | |
| "learning_rate": 4.105337954478756e-06, | |
| "logits/chosen": -2.9261674880981445, | |
| "logits/rejected": -2.7930915355682373, | |
| "logps/chosen": -369.94781494140625, | |
| "logps/rejected": -238.9222869873047, | |
| "loss": 0.6532, | |
| "positive_losses": 0.4402889311313629, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.21584495902061462, | |
| "rewards/margins": 0.2518808841705322, | |
| "rewards/margins_max": 0.37274229526519775, | |
| "rewards/margins_min": 0.1310194730758667, | |
| "rewards/margins_std": 0.1709238588809967, | |
| "rewards/rejected": -0.0360359326004982, | |
| "step": 240 | |
| }, | |
| { | |
| "dpo_losses": 0.6088570356369019, | |
| "epoch": 0.37, | |
| "grad_norm": 7.483775073000372, | |
| "learning_rate": 4.005387349532697e-06, | |
| "logits/chosen": -2.9306282997131348, | |
| "logits/rejected": -2.87394642829895, | |
| "logps/chosen": -293.2420654296875, | |
| "logps/rejected": -272.403076171875, | |
| "loss": 0.6241, | |
| "positive_losses": 0.6371370553970337, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.14793848991394043, | |
| "rewards/margins": 0.18766427040100098, | |
| "rewards/margins_max": 0.2927020192146301, | |
| "rewards/margins_min": 0.08262647688388824, | |
| "rewards/margins_std": 0.14854584634304047, | |
| "rewards/rejected": -0.039725758135318756, | |
| "step": 250 | |
| }, | |
| { | |
| "dpo_losses": 0.630215048789978, | |
| "epoch": 0.38, | |
| "grad_norm": 13.905695323383071, | |
| "learning_rate": 3.901509360874515e-06, | |
| "logits/chosen": -2.832815170288086, | |
| "logits/rejected": -2.8140697479248047, | |
| "logps/chosen": -197.61141967773438, | |
| "logps/rejected": -194.48846435546875, | |
| "loss": 0.6319, | |
| "positive_losses": 0.17327670753002167, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.15451067686080933, | |
| "rewards/margins": 0.14662909507751465, | |
| "rewards/margins_max": 0.2917208671569824, | |
| "rewards/margins_min": 0.0015373497735708952, | |
| "rewards/margins_std": 0.2051907330751419, | |
| "rewards/rejected": 0.00788155198097229, | |
| "step": 260 | |
| }, | |
| { | |
| "dpo_losses": 0.6374494433403015, | |
| "epoch": 0.39, | |
| "grad_norm": 10.795361999407069, | |
| "learning_rate": 3.793974994315991e-06, | |
| "logits/chosen": -2.788649797439575, | |
| "logits/rejected": -2.7856884002685547, | |
| "logps/chosen": -167.8430938720703, | |
| "logps/rejected": -188.2590789794922, | |
| "loss": 0.6446, | |
| "positive_losses": 0.739077091217041, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.13068287074565887, | |
| "rewards/margins": 0.11988089978694916, | |
| "rewards/margins_max": 0.2015564739704132, | |
| "rewards/margins_min": 0.0382053516805172, | |
| "rewards/margins_std": 0.11550667136907578, | |
| "rewards/rejected": 0.010801966302096844, | |
| "step": 270 | |
| }, | |
| { | |
| "dpo_losses": 0.601833701133728, | |
| "epoch": 0.41, | |
| "grad_norm": 1.9286233539251623, | |
| "learning_rate": 3.68306479474137e-06, | |
| "logits/chosen": -3.023601531982422, | |
| "logits/rejected": -2.880030632019043, | |
| "logps/chosen": -352.0651550292969, | |
| "logps/rejected": -203.5298309326172, | |
| "loss": 0.6428, | |
| "positive_losses": 0.44579315185546875, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.21998050808906555, | |
| "rewards/margins": 0.2220487892627716, | |
| "rewards/margins_max": 0.3289792239665985, | |
| "rewards/margins_min": 0.11511830985546112, | |
| "rewards/margins_std": 0.15122249722480774, | |
| "rewards/rejected": -0.002068266272544861, | |
| "step": 280 | |
| }, | |
| { | |
| "dpo_losses": 0.6319935321807861, | |
| "epoch": 0.42, | |
| "grad_norm": 2.944032588280805, | |
| "learning_rate": 3.569068114197784e-06, | |
| "logits/chosen": -2.928798198699951, | |
| "logits/rejected": -2.8589279651641846, | |
| "logps/chosen": -206.49636840820312, | |
| "logps/rejected": -176.32420349121094, | |
| "loss": 0.6206, | |
| "positive_losses": 0.1757659912109375, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": 0.161897212266922, | |
| "rewards/margins": 0.13675031065940857, | |
| "rewards/margins_max": 0.2698245942592621, | |
| "rewards/margins_min": 0.003676059888675809, | |
| "rewards/margins_std": 0.18819543719291687, | |
| "rewards/rejected": 0.02514689229428768, | |
| "step": 290 | |
| }, | |
| { | |
| "dpo_losses": 0.5872830748558044, | |
| "epoch": 0.44, | |
| "grad_norm": 8.216053799669432, | |
| "learning_rate": 3.4522823570088073e-06, | |
| "logits/chosen": -2.855262279510498, | |
| "logits/rejected": -2.8310704231262207, | |
| "logps/chosen": -240.1536407470703, | |
| "logps/rejected": -225.31997680664062, | |
| "loss": 0.6493, | |
| "positive_losses": 0.42629069089889526, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.14640358090400696, | |
| "rewards/margins": 0.24259230494499207, | |
| "rewards/margins_max": 0.3872792422771454, | |
| "rewards/margins_min": 0.09790538251399994, | |
| "rewards/margins_std": 0.20461821556091309, | |
| "rewards/rejected": -0.0961887389421463, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "eval_dpo_losses": 0.6609499454498291, | |
| "eval_logits/chosen": -2.8105618953704834, | |
| "eval_logits/rejected": -2.7656409740448, | |
| "eval_logps/chosen": -268.21075439453125, | |
| "eval_logps/rejected": -249.68173217773438, | |
| "eval_loss": 0.7585510015487671, | |
| "eval_positive_losses": 0.735747218132019, | |
| "eval_rewards/accuracies": 0.6150793433189392, | |
| "eval_rewards/chosen": 0.17010442912578583, | |
| "eval_rewards/margins": 0.0750950425863266, | |
| "eval_rewards/margins_max": 0.2765759229660034, | |
| "eval_rewards/margins_min": -0.10647378861904144, | |
| "eval_rewards/margins_std": 0.17135697603225708, | |
| "eval_rewards/rejected": 0.09500937908887863, | |
| "eval_runtime": 283.2506, | |
| "eval_samples_per_second": 7.061, | |
| "eval_steps_per_second": 0.222, | |
| "step": 300 | |
| }, | |
| { | |
| "dpo_losses": 0.6002415418624878, | |
| "epoch": 0.45, | |
| "grad_norm": 15.987450446439036, | |
| "learning_rate": 3.333012203880528e-06, | |
| "logits/chosen": -2.9334702491760254, | |
| "logits/rejected": -2.8857316970825195, | |
| "logps/chosen": -221.07736206054688, | |
| "logps/rejected": -165.5961456298828, | |
| "loss": 0.6259, | |
| "positive_losses": 0.29423028230667114, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.20694272220134735, | |
| "rewards/margins": 0.22076299786567688, | |
| "rewards/margins_max": 0.40377649664878845, | |
| "rewards/margins_min": 0.03774946928024292, | |
| "rewards/margins_std": 0.25882020592689514, | |
| "rewards/rejected": -0.013820228166878223, | |
| "step": 310 | |
| }, | |
| { | |
| "dpo_losses": 0.5651403069496155, | |
| "epoch": 0.47, | |
| "grad_norm": 41.49020086210869, | |
| "learning_rate": 3.2115688170243735e-06, | |
| "logits/chosen": -2.916748523712158, | |
| "logits/rejected": -2.9150567054748535, | |
| "logps/chosen": -294.4731750488281, | |
| "logps/rejected": -308.61346435546875, | |
| "loss": 0.6257, | |
| "positive_losses": 0.09651489555835724, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.2138613760471344, | |
| "rewards/margins": 0.30642035603523254, | |
| "rewards/margins_max": 0.4777792990207672, | |
| "rewards/margins_min": 0.13506139814853668, | |
| "rewards/margins_std": 0.24233810603618622, | |
| "rewards/rejected": -0.09255897253751755, | |
| "step": 320 | |
| }, | |
| { | |
| "dpo_losses": 0.6225263476371765, | |
| "epoch": 0.48, | |
| "grad_norm": 10.387168274872021, | |
| "learning_rate": 3.0882690283704355e-06, | |
| "logits/chosen": -2.800654649734497, | |
| "logits/rejected": -2.744640588760376, | |
| "logps/chosen": -230.0958709716797, | |
| "logps/rejected": -211.9070587158203, | |
| "loss": 0.6505, | |
| "positive_losses": 0.48747172951698303, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.16328363120555878, | |
| "rewards/margins": 0.16463473439216614, | |
| "rewards/margins_max": 0.3351263403892517, | |
| "rewards/margins_min": -0.005856870673596859, | |
| "rewards/margins_std": 0.24111154675483704, | |
| "rewards/rejected": -0.001351101673208177, | |
| "step": 330 | |
| }, | |
| { | |
| "dpo_losses": 0.5763748288154602, | |
| "epoch": 0.5, | |
| "grad_norm": 11.15788842573243, | |
| "learning_rate": 2.9634345129891296e-06, | |
| "logits/chosen": -2.82387638092041, | |
| "logits/rejected": -2.72804594039917, | |
| "logps/chosen": -288.6628112792969, | |
| "logps/rejected": -272.2504577636719, | |
| "loss": 0.6348, | |
| "positive_losses": 0.106109619140625, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.21110180020332336, | |
| "rewards/margins": 0.2855163514614105, | |
| "rewards/margins_max": 0.5410599708557129, | |
| "rewards/margins_min": 0.029972758144140244, | |
| "rewards/margins_std": 0.36139318346977234, | |
| "rewards/rejected": -0.07441455870866776, | |
| "step": 340 | |
| }, | |
| { | |
| "dpo_losses": 0.6027976870536804, | |
| "epoch": 0.51, | |
| "grad_norm": 15.405775577444263, | |
| "learning_rate": 2.8373909498776746e-06, | |
| "logits/chosen": -2.9628233909606934, | |
| "logits/rejected": -2.954535961151123, | |
| "logps/chosen": -264.20440673828125, | |
| "logps/rejected": -265.29058837890625, | |
| "loss": 0.6444, | |
| "positive_losses": 0.14543990790843964, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.18934306502342224, | |
| "rewards/margins": 0.20881037414073944, | |
| "rewards/margins_max": 0.3755717873573303, | |
| "rewards/margins_min": 0.04204897955060005, | |
| "rewards/margins_std": 0.2358362227678299, | |
| "rewards/rejected": -0.019467316567897797, | |
| "step": 350 | |
| }, | |
| { | |
| "dpo_losses": 0.5799789428710938, | |
| "epoch": 0.53, | |
| "grad_norm": 2.0183316768741295, | |
| "learning_rate": 2.710467172300768e-06, | |
| "logits/chosen": -2.8956894874572754, | |
| "logits/rejected": -2.8347909450531006, | |
| "logps/chosen": -329.68585205078125, | |
| "logps/rejected": -321.13037109375, | |
| "loss": 0.6196, | |
| "positive_losses": 0.2545227110385895, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.23316507041454315, | |
| "rewards/margins": 0.2625313103199005, | |
| "rewards/margins_max": 0.398255318403244, | |
| "rewards/margins_min": 0.12680727243423462, | |
| "rewards/margins_std": 0.19194276630878448, | |
| "rewards/rejected": -0.029366234317421913, | |
| "step": 360 | |
| }, | |
| { | |
| "dpo_losses": 0.5937001705169678, | |
| "epoch": 0.54, | |
| "grad_norm": 12.06051129169852, | |
| "learning_rate": 2.582994309902146e-06, | |
| "logits/chosen": -2.884967565536499, | |
| "logits/rejected": -2.743847370147705, | |
| "logps/chosen": -285.45123291015625, | |
| "logps/rejected": -247.1053466796875, | |
| "loss": 0.6401, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.1828571856021881, | |
| "rewards/margins": 0.22877776622772217, | |
| "rewards/margins_max": 0.3255845904350281, | |
| "rewards/margins_min": 0.13197092711925507, | |
| "rewards/margins_std": 0.13690553605556488, | |
| "rewards/rejected": -0.04592058062553406, | |
| "step": 370 | |
| }, | |
| { | |
| "dpo_losses": 0.595887303352356, | |
| "epoch": 0.56, | |
| "grad_norm": 3.3756204670545618, | |
| "learning_rate": 2.4553049248251512e-06, | |
| "logits/chosen": -2.788328170776367, | |
| "logits/rejected": -2.836977958679199, | |
| "logps/chosen": -214.0673065185547, | |
| "logps/rejected": -235.258544921875, | |
| "loss": 0.5859, | |
| "positive_losses": 0.17899170517921448, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.1966702789068222, | |
| "rewards/margins": 0.2272748053073883, | |
| "rewards/margins_max": 0.33700865507125854, | |
| "rewards/margins_min": 0.11754089593887329, | |
| "rewards/margins_std": 0.15518715977668762, | |
| "rewards/rejected": -0.03060450591146946, | |
| "step": 380 | |
| }, | |
| { | |
| "dpo_losses": 0.6092433333396912, | |
| "epoch": 0.57, | |
| "grad_norm": 2.5893672084668107, | |
| "learning_rate": 2.3277321440960733e-06, | |
| "logits/chosen": -2.9706382751464844, | |
| "logits/rejected": -2.9543769359588623, | |
| "logps/chosen": -256.9505615234375, | |
| "logps/rejected": -262.0420227050781, | |
| "loss": 0.6297, | |
| "positive_losses": 0.7152191400527954, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": 0.16275997459888458, | |
| "rewards/margins": 0.20270125567913055, | |
| "rewards/margins_max": 0.401836633682251, | |
| "rewards/margins_min": 0.0035658925771713257, | |
| "rewards/margins_std": 0.28161993622779846, | |
| "rewards/rejected": -0.039941295981407166, | |
| "step": 390 | |
| }, | |
| { | |
| "dpo_losses": 0.5526755452156067, | |
| "epoch": 0.58, | |
| "grad_norm": 21.63358202743805, | |
| "learning_rate": 2.20060879053377e-06, | |
| "logits/chosen": -2.8260955810546875, | |
| "logits/rejected": -2.7942147254943848, | |
| "logps/chosen": -184.72544860839844, | |
| "logps/rejected": -232.3735809326172, | |
| "loss": 0.6224, | |
| "positive_losses": 0.5248996019363403, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.16737225651741028, | |
| "rewards/margins": 0.3254551887512207, | |
| "rewards/margins_max": 0.506883442401886, | |
| "rewards/margins_min": 0.14402692019939423, | |
| "rewards/margins_std": 0.256578266620636, | |
| "rewards/rejected": -0.15808293223381042, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "eval_dpo_losses": 0.6528598070144653, | |
| "eval_logits/chosen": -2.819880723953247, | |
| "eval_logits/rejected": -2.7766764163970947, | |
| "eval_logps/chosen": -274.035888671875, | |
| "eval_logps/rejected": -258.0920715332031, | |
| "eval_loss": 0.9942816495895386, | |
| "eval_positive_losses": 3.3746726512908936, | |
| "eval_rewards/accuracies": 0.6388888955116272, | |
| "eval_rewards/chosen": 0.11185282468795776, | |
| "eval_rewards/margins": 0.10094699263572693, | |
| "eval_rewards/margins_max": 0.38355645537376404, | |
| "eval_rewards/margins_min": -0.1620650738477707, | |
| "eval_rewards/margins_std": 0.24338804185390472, | |
| "eval_rewards/rejected": 0.01090583112090826, | |
| "eval_runtime": 283.2899, | |
| "eval_samples_per_second": 7.06, | |
| "eval_steps_per_second": 0.222, | |
| "step": 400 | |
| }, | |
| { | |
| "dpo_losses": 0.531283438205719, | |
| "epoch": 0.6, | |
| "grad_norm": 11.192408672928963, | |
| "learning_rate": 2.0742665144529374e-06, | |
| "logits/chosen": -2.8772964477539062, | |
| "logits/rejected": -2.7996630668640137, | |
| "logps/chosen": -310.952392578125, | |
| "logps/rejected": -263.61370849609375, | |
| "loss": 0.6614, | |
| "positive_losses": 0.6912002563476562, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.3101710379123688, | |
| "rewards/margins": 0.4205913543701172, | |
| "rewards/margins_max": 0.6372691988945007, | |
| "rewards/margins_min": 0.20391342043876648, | |
| "rewards/margins_std": 0.30642884969711304, | |
| "rewards/rejected": -0.11042030900716782, | |
| "step": 410 | |
| }, | |
| { | |
| "dpo_losses": 0.5354982614517212, | |
| "epoch": 0.61, | |
| "grad_norm": 15.45856376263706, | |
| "learning_rate": 1.9490349284263036e-06, | |
| "logits/chosen": -2.8157570362091064, | |
| "logits/rejected": -2.758150577545166, | |
| "logps/chosen": -286.8282775878906, | |
| "logps/rejected": -253.0894775390625, | |
| "loss": 0.6483, | |
| "positive_losses": 0.7530021667480469, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.2359578162431717, | |
| "rewards/margins": 0.3807125985622406, | |
| "rewards/margins_max": 0.5745470523834229, | |
| "rewards/margins_min": 0.18687808513641357, | |
| "rewards/margins_std": 0.27412334084510803, | |
| "rewards/rejected": -0.14475473761558533, | |
| "step": 420 | |
| }, | |
| { | |
| "dpo_losses": 0.5478672385215759, | |
| "epoch": 0.63, | |
| "grad_norm": 9.397460886758937, | |
| "learning_rate": 1.8252407473630606e-06, | |
| "logits/chosen": -2.981616735458374, | |
| "logits/rejected": -2.9860920906066895, | |
| "logps/chosen": -268.0840148925781, | |
| "logps/rejected": -284.7814025878906, | |
| "loss": 0.6187, | |
| "positive_losses": 0.08653469383716583, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.19860908389091492, | |
| "rewards/margins": 0.35602104663848877, | |
| "rewards/margins_max": 0.5557164549827576, | |
| "rewards/margins_min": 0.15632562339305878, | |
| "rewards/margins_std": 0.28241199254989624, | |
| "rewards/rejected": -0.15741200745105743, | |
| "step": 430 | |
| }, | |
| { | |
| "dpo_losses": 0.5760594010353088, | |
| "epoch": 0.64, | |
| "grad_norm": 23.342372051867663, | |
| "learning_rate": 1.7032069361469765e-06, | |
| "logits/chosen": -2.775434732437134, | |
| "logits/rejected": -2.77290678024292, | |
| "logps/chosen": -213.5185089111328, | |
| "logps/rejected": -278.1703186035156, | |
| "loss": 0.6391, | |
| "positive_losses": 0.018738174811005592, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": 0.19827936589717865, | |
| "rewards/margins": 0.2895718812942505, | |
| "rewards/margins_max": 0.5261528491973877, | |
| "rewards/margins_min": 0.05299092084169388, | |
| "rewards/margins_std": 0.3345760405063629, | |
| "rewards/rejected": -0.09129253774881363, | |
| "step": 440 | |
| }, | |
| { | |
| "dpo_losses": 0.5661223530769348, | |
| "epoch": 0.66, | |
| "grad_norm": 2.1591786937116098, | |
| "learning_rate": 1.5832518670578802e-06, | |
| "logits/chosen": -3.0003552436828613, | |
| "logits/rejected": -2.9453907012939453, | |
| "logps/chosen": -272.0791015625, | |
| "logps/rejected": -293.05645751953125, | |
| "loss": 0.6369, | |
| "positive_losses": 0.25814515352249146, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.2058447152376175, | |
| "rewards/margins": 0.2981029152870178, | |
| "rewards/margins_max": 0.49641966819763184, | |
| "rewards/margins_min": 0.099786177277565, | |
| "rewards/margins_std": 0.28046220541000366, | |
| "rewards/rejected": -0.09225818514823914, | |
| "step": 450 | |
| }, | |
| { | |
| "dpo_losses": 0.5694680213928223, | |
| "epoch": 0.67, | |
| "grad_norm": 2.6035617787075602, | |
| "learning_rate": 1.4656884891747398e-06, | |
| "logits/chosen": -2.813758373260498, | |
| "logits/rejected": -2.838399887084961, | |
| "logps/chosen": -266.3056945800781, | |
| "logps/rejected": -263.6679382324219, | |
| "loss": 0.6275, | |
| "positive_losses": 0.17335128784179688, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.19134476780891418, | |
| "rewards/margins": 0.3021687865257263, | |
| "rewards/margins_max": 0.5000616312026978, | |
| "rewards/margins_min": 0.10427598655223846, | |
| "rewards/margins_std": 0.2798627018928528, | |
| "rewards/rejected": -0.11082406342029572, | |
| "step": 460 | |
| }, | |
| { | |
| "dpo_losses": 0.6458374261856079, | |
| "epoch": 0.69, | |
| "grad_norm": 2.5506219741141543, | |
| "learning_rate": 1.3508235119272466e-06, | |
| "logits/chosen": -2.8581721782684326, | |
| "logits/rejected": -2.8341779708862305, | |
| "logps/chosen": -254.4308319091797, | |
| "logps/rejected": -253.99722290039062, | |
| "loss": 0.6175, | |
| "positive_losses": 2.3088302612304688, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": 0.11066363006830215, | |
| "rewards/margins": 0.1266460120677948, | |
| "rewards/margins_max": 0.2916102409362793, | |
| "rewards/margins_min": -0.0383182056248188, | |
| "rewards/margins_std": 0.23329463601112366, | |
| "rewards/rejected": -0.01598239876329899, | |
| "step": 470 | |
| }, | |
| { | |
| "dpo_losses": 0.5230454206466675, | |
| "epoch": 0.7, | |
| "grad_norm": 24.43956168373256, | |
| "learning_rate": 1.238956604925934e-06, | |
| "logits/chosen": -2.7936887741088867, | |
| "logits/rejected": -2.7815768718719482, | |
| "logps/chosen": -241.1090850830078, | |
| "logps/rejected": -284.8046569824219, | |
| "loss": 0.6176, | |
| "positive_losses": 0.7508819699287415, | |
| "rewards/accuracies": 0.8999999761581421, | |
| "rewards/chosen": 0.23765699565410614, | |
| "rewards/margins": 0.422058641910553, | |
| "rewards/margins_max": 0.6369160413742065, | |
| "rewards/margins_min": 0.20720121264457703, | |
| "rewards/margins_std": 0.3038543164730072, | |
| "rewards/rejected": -0.18440163135528564, | |
| "step": 480 | |
| }, | |
| { | |
| "dpo_losses": 0.5921580195426941, | |
| "epoch": 0.72, | |
| "grad_norm": 9.14828905462399, | |
| "learning_rate": 1.1303796161583763e-06, | |
| "logits/chosen": -2.953059673309326, | |
| "logits/rejected": -2.935351848602295, | |
| "logps/chosen": -277.16998291015625, | |
| "logps/rejected": -322.24908447265625, | |
| "loss": 0.5856, | |
| "positive_losses": 0.24810238182544708, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.18392740190029144, | |
| "rewards/margins": 0.252055287361145, | |
| "rewards/margins_max": 0.42164430022239685, | |
| "rewards/margins_min": 0.08246620744466782, | |
| "rewards/margins_std": 0.23983514308929443, | |
| "rewards/rejected": -0.06812787055969238, | |
| "step": 490 | |
| }, | |
| { | |
| "dpo_losses": 0.5570933222770691, | |
| "epoch": 0.73, | |
| "grad_norm": 26.824091044275153, | |
| "learning_rate": 1.0253758105911169e-06, | |
| "logits/chosen": -2.970041275024414, | |
| "logits/rejected": -2.90769362449646, | |
| "logps/chosen": -304.5309143066406, | |
| "logps/rejected": -356.88641357421875, | |
| "loss": 0.5674, | |
| "positive_losses": 0.19576720893383026, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.23450036346912384, | |
| "rewards/margins": 0.35680580139160156, | |
| "rewards/margins_max": 0.5582734942436218, | |
| "rewards/margins_min": 0.15533806383609772, | |
| "rewards/margins_std": 0.2849184274673462, | |
| "rewards/rejected": -0.12230543792247772, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "eval_dpo_losses": 0.6564862728118896, | |
| "eval_logits/chosen": -2.8376405239105225, | |
| "eval_logits/rejected": -2.793374538421631, | |
| "eval_logps/chosen": -278.80975341796875, | |
| "eval_logps/rejected": -262.5242004394531, | |
| "eval_loss": 1.1831417083740234, | |
| "eval_positive_losses": 5.736485481262207, | |
| "eval_rewards/accuracies": 0.6269841194152832, | |
| "eval_rewards/chosen": 0.06411468237638474, | |
| "eval_rewards/margins": 0.09753014147281647, | |
| "eval_rewards/margins_max": 0.41429367661476135, | |
| "eval_rewards/margins_min": -0.18840637803077698, | |
| "eval_rewards/margins_std": 0.27024951577186584, | |
| "eval_rewards/rejected": -0.033415455371141434, | |
| "eval_runtime": 283.1365, | |
| "eval_samples_per_second": 7.064, | |
| "eval_steps_per_second": 0.223, | |
| "step": 500 | |
| }, | |
| { | |
| "dpo_losses": 0.5341383218765259, | |
| "epoch": 0.75, | |
| "grad_norm": 2.630596915267566, | |
| "learning_rate": 9.24219131163705e-07, | |
| "logits/chosen": -2.8331634998321533, | |
| "logits/rejected": -2.8200929164886475, | |
| "logps/chosen": -288.0255126953125, | |
| "logps/rejected": -313.35601806640625, | |
| "loss": 0.6361, | |
| "positive_losses": 1.7603362798690796, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.21840150654315948, | |
| "rewards/margins": 0.4240838885307312, | |
| "rewards/margins_max": 0.7840636968612671, | |
| "rewards/margins_min": 0.06410404294729233, | |
| "rewards/margins_std": 0.509088397026062, | |
| "rewards/rejected": -0.2056823968887329, | |
| "step": 510 | |
| }, | |
| { | |
| "dpo_losses": 0.49555259943008423, | |
| "epoch": 0.76, | |
| "grad_norm": 13.21991755241966, | |
| "learning_rate": 8.271734841028553e-07, | |
| "logits/chosen": -2.72208833694458, | |
| "logits/rejected": -2.6312568187713623, | |
| "logps/chosen": -286.3044128417969, | |
| "logps/rejected": -246.79788208007812, | |
| "loss": 0.6459, | |
| "positive_losses": 0.1013515442609787, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.28137513995170593, | |
| "rewards/margins": 0.4984091818332672, | |
| "rewards/margins_max": 0.7110401391983032, | |
| "rewards/margins_min": 0.28577831387519836, | |
| "rewards/margins_std": 0.3007054626941681, | |
| "rewards/rejected": -0.21703402698040009, | |
| "step": 520 | |
| }, | |
| { | |
| "dpo_losses": 0.5471224188804626, | |
| "epoch": 0.77, | |
| "grad_norm": 7.226241019031729, | |
| "learning_rate": 7.344920504212244e-07, | |
| "logits/chosen": -2.9117705821990967, | |
| "logits/rejected": -2.8744723796844482, | |
| "logps/chosen": -206.4365997314453, | |
| "logps/rejected": -208.7721710205078, | |
| "loss": 0.6331, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 0.20657558739185333, | |
| "rewards/margins": 0.34465348720550537, | |
| "rewards/margins_max": 0.5002694725990295, | |
| "rewards/margins_min": 0.1890375316143036, | |
| "rewards/margins_std": 0.22007422149181366, | |
| "rewards/rejected": -0.13807791471481323, | |
| "step": 530 | |
| }, | |
| { | |
| "dpo_losses": 0.5800082087516785, | |
| "epoch": 0.79, | |
| "grad_norm": 48.796149530310025, | |
| "learning_rate": 6.464166253970672e-07, | |
| "logits/chosen": -2.8512115478515625, | |
| "logits/rejected": -2.8704025745391846, | |
| "logps/chosen": -307.69915771484375, | |
| "logps/rejected": -285.3216857910156, | |
| "loss": 0.5932, | |
| "positive_losses": 0.1961948424577713, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.1846921294927597, | |
| "rewards/margins": 0.2769491672515869, | |
| "rewards/margins_max": 0.47591620683670044, | |
| "rewards/margins_min": 0.077982097864151, | |
| "rewards/margins_std": 0.28138190507888794, | |
| "rewards/rejected": -0.09225703775882721, | |
| "step": 540 | |
| }, | |
| { | |
| "dpo_losses": 0.5755653381347656, | |
| "epoch": 0.8, | |
| "grad_norm": 2.795078318527347, | |
| "learning_rate": 5.631769877579535e-07, | |
| "logits/chosen": -2.9338223934173584, | |
| "logits/rejected": -2.885927200317383, | |
| "logps/chosen": -237.5154266357422, | |
| "logps/rejected": -245.45254516601562, | |
| "loss": 0.6043, | |
| "positive_losses": 1.3403419256210327, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.13946720957756042, | |
| "rewards/margins": 0.28026267886161804, | |
| "rewards/margins_max": 0.4180312752723694, | |
| "rewards/margins_min": 0.1424940675497055, | |
| "rewards/margins_std": 0.19483418762683868, | |
| "rewards/rejected": -0.14079545438289642, | |
| "step": 550 | |
| }, | |
| { | |
| "dpo_losses": 0.5164459943771362, | |
| "epoch": 0.82, | |
| "grad_norm": 3.090907013989327, | |
| "learning_rate": 4.849903002143114e-07, | |
| "logits/chosen": -3.0459659099578857, | |
| "logits/rejected": -2.9688525199890137, | |
| "logps/chosen": -348.83740234375, | |
| "logps/rejected": -346.1819763183594, | |
| "loss": 0.5621, | |
| "positive_losses": 0.1016082763671875, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.23445944488048553, | |
| "rewards/margins": 0.4368625283241272, | |
| "rewards/margins_max": 0.6422832012176514, | |
| "rewards/margins_min": 0.23144181072711945, | |
| "rewards/margins_std": 0.2905087471008301, | |
| "rewards/rejected": -0.20240306854248047, | |
| "step": 560 | |
| }, | |
| { | |
| "dpo_losses": 0.5176469087600708, | |
| "epoch": 0.83, | |
| "grad_norm": 7.846420016024575, | |
| "learning_rate": 4.1206054290670537e-07, | |
| "logits/chosen": -2.894327163696289, | |
| "logits/rejected": -2.896099805831909, | |
| "logps/chosen": -234.593994140625, | |
| "logps/rejected": -323.6400146484375, | |
| "loss": 0.6136, | |
| "positive_losses": 1.2456893920898438, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.17635245621204376, | |
| "rewards/margins": 0.4464770257472992, | |
| "rewards/margins_max": 0.6318201422691345, | |
| "rewards/margins_min": 0.2611338794231415, | |
| "rewards/margins_std": 0.2621147930622101, | |
| "rewards/rejected": -0.27012452483177185, | |
| "step": 570 | |
| }, | |
| { | |
| "dpo_losses": 0.619495689868927, | |
| "epoch": 0.85, | |
| "grad_norm": 47.00247250810323, | |
| "learning_rate": 3.44577981244944e-07, | |
| "logits/chosen": -2.9653825759887695, | |
| "logits/rejected": -2.980112314224243, | |
| "logps/chosen": -235.0575408935547, | |
| "logps/rejected": -250.17593383789062, | |
| "loss": 0.6852, | |
| "positive_losses": 1.9925556182861328, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": 0.1124948263168335, | |
| "rewards/margins": 0.1802656650543213, | |
| "rewards/margins_max": 0.35439062118530273, | |
| "rewards/margins_min": 0.006140687968581915, | |
| "rewards/margins_std": 0.24624991416931152, | |
| "rewards/rejected": -0.06777085363864899, | |
| "step": 580 | |
| }, | |
| { | |
| "dpo_losses": 0.5588719248771667, | |
| "epoch": 0.86, | |
| "grad_norm": 4.545382225154878, | |
| "learning_rate": 2.827186695273482e-07, | |
| "logits/chosen": -3.0688188076019287, | |
| "logits/rejected": -2.9839656352996826, | |
| "logps/chosen": -374.6413269042969, | |
| "logps/rejected": -343.8844909667969, | |
| "loss": 0.6152, | |
| "positive_losses": 0.27655029296875, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": 0.2746116518974304, | |
| "rewards/margins": 0.3599032461643219, | |
| "rewards/margins_max": 0.609930157661438, | |
| "rewards/margins_min": 0.10987631976604462, | |
| "rewards/margins_std": 0.3535914719104767, | |
| "rewards/rejected": -0.08529156446456909, | |
| "step": 590 | |
| }, | |
| { | |
| "dpo_losses": 0.5024896860122681, | |
| "epoch": 0.88, | |
| "grad_norm": 10.429875129983618, | |
| "learning_rate": 2.2664399163518786e-07, | |
| "logits/chosen": -2.891846179962158, | |
| "logits/rejected": -2.812058925628662, | |
| "logps/chosen": -292.8556213378906, | |
| "logps/rejected": -254.47628784179688, | |
| "loss": 0.5749, | |
| "positive_losses": 0.0, | |
| "rewards/accuracies": 0.949999988079071, | |
| "rewards/chosen": 0.27775660157203674, | |
| "rewards/margins": 0.49465060234069824, | |
| "rewards/margins_max": 0.8027955293655396, | |
| "rewards/margins_min": 0.18650567531585693, | |
| "rewards/margins_std": 0.4357827305793762, | |
| "rewards/rejected": -0.2168940007686615, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "eval_dpo_losses": 0.6511540412902832, | |
| "eval_logits/chosen": -2.8279476165771484, | |
| "eval_logits/rejected": -2.7839229106903076, | |
| "eval_logps/chosen": -274.8698425292969, | |
| "eval_logps/rejected": -259.9163513183594, | |
| "eval_loss": 1.0992192029953003, | |
| "eval_positive_losses": 4.5979323387146, | |
| "eval_rewards/accuracies": 0.6190476417541504, | |
| "eval_rewards/chosen": 0.1035134568810463, | |
| "eval_rewards/margins": 0.11085036396980286, | |
| "eval_rewards/margins_max": 0.4368113875389099, | |
| "eval_rewards/margins_min": -0.18844377994537354, | |
| "eval_rewards/margins_std": 0.2790098786354065, | |
| "eval_rewards/rejected": -0.0073369028978049755, | |
| "eval_runtime": 283.0506, | |
| "eval_samples_per_second": 7.066, | |
| "eval_steps_per_second": 0.223, | |
| "step": 600 | |
| }, | |
| { | |
| "dpo_losses": 0.5908172726631165, | |
| "epoch": 0.89, | |
| "grad_norm": 8.899268268122263, | |
| "learning_rate": 1.7650024000056415e-07, | |
| "logits/chosen": -2.910013198852539, | |
| "logits/rejected": -2.8941874504089355, | |
| "logps/chosen": -202.37757873535156, | |
| "logps/rejected": -226.99508666992188, | |
| "loss": 0.6342, | |
| "positive_losses": 1.9391590356826782, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.1791270673274994, | |
| "rewards/margins": 0.25547030568122864, | |
| "rewards/margins_max": 0.4546757638454437, | |
| "rewards/margins_min": 0.05626480653882027, | |
| "rewards/margins_std": 0.2817191183567047, | |
| "rewards/rejected": -0.07634319365024567, | |
| "step": 610 | |
| }, | |
| { | |
| "dpo_losses": 0.5537667870521545, | |
| "epoch": 0.91, | |
| "grad_norm": 27.10072020527372, | |
| "learning_rate": 1.324182339461544e-07, | |
| "logits/chosen": -2.87471342086792, | |
| "logits/rejected": -2.8323957920074463, | |
| "logps/chosen": -255.8001251220703, | |
| "logps/rejected": -216.3787841796875, | |
| "loss": 0.6711, | |
| "positive_losses": 0.1423923522233963, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.22555597126483917, | |
| "rewards/margins": 0.34654873609542847, | |
| "rewards/margins_max": 0.5156995058059692, | |
| "rewards/margins_min": 0.17739805579185486, | |
| "rewards/margins_std": 0.23921525478363037, | |
| "rewards/rejected": -0.12099279463291168, | |
| "step": 620 | |
| }, | |
| { | |
| "dpo_losses": 0.48896676301956177, | |
| "epoch": 0.92, | |
| "grad_norm": 2.6767258362110957, | |
| "learning_rate": 9.451297839253915e-08, | |
| "logits/chosen": -2.836315393447876, | |
| "logits/rejected": -2.7731261253356934, | |
| "logps/chosen": -304.1548156738281, | |
| "logps/rejected": -336.2890319824219, | |
| "loss": 0.6356, | |
| "positive_losses": 0.17710499465465546, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.31015846133232117, | |
| "rewards/margins": 0.5357273817062378, | |
| "rewards/margins_max": 0.8079883456230164, | |
| "rewards/margins_min": 0.263466477394104, | |
| "rewards/margins_std": 0.38503509759902954, | |
| "rewards/rejected": -0.2255689650774002, | |
| "step": 630 | |
| }, | |
| { | |
| "dpo_losses": 0.5176305174827576, | |
| "epoch": 0.94, | |
| "grad_norm": 18.280815132381182, | |
| "learning_rate": 6.288336382349463e-08, | |
| "logits/chosen": -2.8430378437042236, | |
| "logits/rejected": -2.7495901584625244, | |
| "logps/chosen": -359.60797119140625, | |
| "logps/rejected": -298.79827880859375, | |
| "loss": 0.5757, | |
| "positive_losses": 1.0459808111190796, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.24247094988822937, | |
| "rewards/margins": 0.44258061051368713, | |
| "rewards/margins_max": 0.6816617846488953, | |
| "rewards/margins_min": 0.20349940657615662, | |
| "rewards/margins_std": 0.33811187744140625, | |
| "rewards/rejected": -0.20010964572429657, | |
| "step": 640 | |
| }, | |
| { | |
| "dpo_losses": 0.5474685430526733, | |
| "epoch": 0.95, | |
| "grad_norm": 2.7207730538280863, | |
| "learning_rate": 3.761190829201067e-08, | |
| "logits/chosen": -2.8519351482391357, | |
| "logits/rejected": -2.805759906768799, | |
| "logps/chosen": -365.9830627441406, | |
| "logps/rejected": -278.6522521972656, | |
| "loss": 0.5966, | |
| "positive_losses": 0.05963592603802681, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.2809067666530609, | |
| "rewards/margins": 0.4035716652870178, | |
| "rewards/margins_max": 0.7429118156433105, | |
| "rewards/margins_min": 0.06423152983188629, | |
| "rewards/margins_std": 0.47989946603775024, | |
| "rewards/rejected": -0.12266488373279572, | |
| "step": 650 | |
| }, | |
| { | |
| "dpo_losses": 0.5853177309036255, | |
| "epoch": 0.96, | |
| "grad_norm": 40.690683079215134, | |
| "learning_rate": 1.876454214011253e-08, | |
| "logits/chosen": -2.9034006595611572, | |
| "logits/rejected": -2.8463714122772217, | |
| "logps/chosen": -243.8646697998047, | |
| "logps/rejected": -225.9236602783203, | |
| "loss": 0.6752, | |
| "positive_losses": 0.5306800603866577, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": 0.21680143475532532, | |
| "rewards/margins": 0.2727014422416687, | |
| "rewards/margins_max": 0.4745156764984131, | |
| "rewards/margins_min": 0.07088717073202133, | |
| "rewards/margins_std": 0.2854084372520447, | |
| "rewards/rejected": -0.05589999631047249, | |
| "step": 660 | |
| }, | |
| { | |
| "dpo_losses": 0.5261252522468567, | |
| "epoch": 0.98, | |
| "grad_norm": 8.821965322055776, | |
| "learning_rate": 6.390435994127753e-09, | |
| "logits/chosen": -2.8224921226501465, | |
| "logits/rejected": -2.8528945446014404, | |
| "logps/chosen": -284.4530944824219, | |
| "logps/rejected": -375.3456115722656, | |
| "loss": 0.6203, | |
| "positive_losses": 0.12337493896484375, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": 0.2311427891254425, | |
| "rewards/margins": 0.4244818687438965, | |
| "rewards/margins_max": 0.6252471208572388, | |
| "rewards/margins_min": 0.2237166464328766, | |
| "rewards/margins_std": 0.2839249074459076, | |
| "rewards/rejected": -0.19333907961845398, | |
| "step": 670 | |
| }, | |
| { | |
| "dpo_losses": 0.5133123397827148, | |
| "epoch": 0.99, | |
| "grad_norm": 23.801829753390283, | |
| "learning_rate": 5.218724841346556e-10, | |
| "logits/chosen": -2.689457416534424, | |
| "logits/rejected": -2.6667604446411133, | |
| "logps/chosen": -358.9125061035156, | |
| "logps/rejected": -298.8949890136719, | |
| "loss": 0.6128, | |
| "positive_losses": 1.849574327468872, | |
| "rewards/accuracies": 0.8500000238418579, | |
| "rewards/chosen": 0.30895963311195374, | |
| "rewards/margins": 0.4615735113620758, | |
| "rewards/margins_max": 0.7685288190841675, | |
| "rewards/margins_min": 0.15461814403533936, | |
| "rewards/margins_std": 0.43410032987594604, | |
| "rewards/rejected": -0.15261384844779968, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 684, | |
| "total_flos": 0.0, | |
| "train_loss": 0.641208895814349, | |
| "train_runtime": 6249.598, | |
| "train_samples_per_second": 1.751, | |
| "train_steps_per_second": 0.109 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 684, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 100, | |
| "total_flos": 0.0, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |