| { | |
| "best_metric": 0.8505691885948181, | |
| "best_model_checkpoint": "saves/Mistral-7B-Instruct-v0.3/lora/orpo-salt-half/checkpoint-1500", | |
| "epoch": 2.9974597798475866, | |
| "eval_steps": 500, | |
| "global_step": 1770, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.01693480101608806, | |
| "grad_norm": 8.384780883789062, | |
| "learning_rate": 4.999614014035063e-06, | |
| "logits/chosen": -2.9335973262786865, | |
| "logits/rejected": -2.9718575477600098, | |
| "logps/chosen": -1.0935328006744385, | |
| "logps/rejected": -1.5320154428482056, | |
| "loss": 1.1602, | |
| "odds_ratio_loss": 0.6671324968338013, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.10935328900814056, | |
| "rewards/margins": 0.04384826496243477, | |
| "rewards/rejected": -0.15320155024528503, | |
| "sft_loss": 1.0935328006744385, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.03386960203217612, | |
| "grad_norm": 5.07673454284668, | |
| "learning_rate": 4.998440543386042e-06, | |
| "logits/chosen": -2.899705648422241, | |
| "logits/rejected": -2.8991312980651855, | |
| "logps/chosen": -1.0815098285675049, | |
| "logps/rejected": -1.5599451065063477, | |
| "loss": 1.1396, | |
| "odds_ratio_loss": 0.5810434818267822, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.10815098136663437, | |
| "rewards/margins": 0.04784352704882622, | |
| "rewards/rejected": -0.1559945046901703, | |
| "sft_loss": 1.0815098285675049, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.05080440304826418, | |
| "grad_norm": 4.302060127258301, | |
| "learning_rate": 4.996479918381253e-06, | |
| "logits/chosen": -2.944632053375244, | |
| "logits/rejected": -2.975834846496582, | |
| "logps/chosen": -1.056970238685608, | |
| "logps/rejected": -1.281690001487732, | |
| "loss": 1.1267, | |
| "odds_ratio_loss": 0.6977876424789429, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.10569703578948975, | |
| "rewards/margins": 0.022471977397799492, | |
| "rewards/rejected": -0.1281690150499344, | |
| "sft_loss": 1.056970238685608, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.06773920406435224, | |
| "grad_norm": 3.221508026123047, | |
| "learning_rate": 4.993732756731818e-06, | |
| "logits/chosen": -2.925711154937744, | |
| "logits/rejected": -2.972975730895996, | |
| "logps/chosen": -0.8646748661994934, | |
| "logps/rejected": -1.3530806303024292, | |
| "loss": 0.9216, | |
| "odds_ratio_loss": 0.5692964792251587, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.08646748960018158, | |
| "rewards/margins": 0.04884058237075806, | |
| "rewards/rejected": -0.13530807197093964, | |
| "sft_loss": 0.8646748661994934, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.0846740050804403, | |
| "grad_norm": 11.413677215576172, | |
| "learning_rate": 4.9901999239537345e-06, | |
| "logits/chosen": -2.9809317588806152, | |
| "logits/rejected": -2.983664035797119, | |
| "logps/chosen": -0.9910508990287781, | |
| "logps/rejected": -1.344020962715149, | |
| "loss": 1.0558, | |
| "odds_ratio_loss": 0.6479853987693787, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.0991050973534584, | |
| "rewards/margins": 0.035297006368637085, | |
| "rewards/rejected": -0.1344021111726761, | |
| "sft_loss": 0.9910508990287781, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.10160880609652836, | |
| "grad_norm": 1.7445276975631714, | |
| "learning_rate": 4.985882533095186e-06, | |
| "logits/chosen": -2.9525933265686035, | |
| "logits/rejected": -2.955658435821533, | |
| "logps/chosen": -0.9359370470046997, | |
| "logps/rejected": -1.2753798961639404, | |
| "loss": 1.0023, | |
| "odds_ratio_loss": 0.6634014248847961, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.09359370172023773, | |
| "rewards/margins": 0.03394431248307228, | |
| "rewards/rejected": -0.12753799557685852, | |
| "sft_loss": 0.9359370470046997, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.11854360711261643, | |
| "grad_norm": 3.4384100437164307, | |
| "learning_rate": 4.9807819443858705e-06, | |
| "logits/chosen": -2.956145763397217, | |
| "logits/rejected": -2.9750401973724365, | |
| "logps/chosen": -0.8652639389038086, | |
| "logps/rejected": -1.2253761291503906, | |
| "loss": 0.9227, | |
| "odds_ratio_loss": 0.5748167634010315, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.08652639389038086, | |
| "rewards/margins": 0.0360112190246582, | |
| "rewards/rejected": -0.12253761291503906, | |
| "sft_loss": 0.8652639389038086, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.1354784081287045, | |
| "grad_norm": 2.6011013984680176, | |
| "learning_rate": 4.9748997648084404e-06, | |
| "logits/chosen": -2.927016496658325, | |
| "logits/rejected": -2.961350917816162, | |
| "logps/chosen": -0.9362251162528992, | |
| "logps/rejected": -1.1120071411132812, | |
| "loss": 1.0068, | |
| "odds_ratio_loss": 0.7062079906463623, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.09362251311540604, | |
| "rewards/margins": 0.017578203231096268, | |
| "rewards/rejected": -0.111200712621212, | |
| "sft_loss": 0.9362251162528992, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.15241320914479256, | |
| "grad_norm": 2.413475751876831, | |
| "learning_rate": 4.96823784759222e-06, | |
| "logits/chosen": -2.869145154953003, | |
| "logits/rejected": -2.916862964630127, | |
| "logps/chosen": -0.8781692385673523, | |
| "logps/rejected": -1.2194924354553223, | |
| "loss": 0.9334, | |
| "odds_ratio_loss": 0.552145779132843, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.08781692385673523, | |
| "rewards/margins": 0.034132327884435654, | |
| "rewards/rejected": -0.12194924056529999, | |
| "sft_loss": 0.8781692385673523, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.1693480101608806, | |
| "grad_norm": 1.9281786680221558, | |
| "learning_rate": 4.960798291629323e-06, | |
| "logits/chosen": -2.9603519439697266, | |
| "logits/rejected": -2.9720394611358643, | |
| "logps/chosen": -0.8728249669075012, | |
| "logps/rejected": -1.0510112047195435, | |
| "loss": 0.9387, | |
| "odds_ratio_loss": 0.658331036567688, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.08728249371051788, | |
| "rewards/margins": 0.017818626016378403, | |
| "rewards/rejected": -0.10510112345218658, | |
| "sft_loss": 0.8728249669075012, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.18628281117696868, | |
| "grad_norm": 2.8832967281341553, | |
| "learning_rate": 4.952583440813383e-06, | |
| "logits/chosen": -2.9769511222839355, | |
| "logits/rejected": -2.991858720779419, | |
| "logps/chosen": -0.9942096471786499, | |
| "logps/rejected": -1.25453519821167, | |
| "loss": 1.0667, | |
| "odds_ratio_loss": 0.7251978516578674, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.09942097961902618, | |
| "rewards/margins": 0.026032552123069763, | |
| "rewards/rejected": -0.12545353174209595, | |
| "sft_loss": 0.9942096471786499, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.20321761219305673, | |
| "grad_norm": 1.2281321287155151, | |
| "learning_rate": 4.943595883301086e-06, | |
| "logits/chosen": -2.9461495876312256, | |
| "logits/rejected": -2.988556385040283, | |
| "logps/chosen": -0.8882713317871094, | |
| "logps/rejected": -1.1232919692993164, | |
| "loss": 0.9522, | |
| "odds_ratio_loss": 0.6390038728713989, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.08882713317871094, | |
| "rewards/margins": 0.02350207045674324, | |
| "rewards/rejected": -0.11232920736074448, | |
| "sft_loss": 0.8882713317871094, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.2201524132091448, | |
| "grad_norm": 2.2648568153381348, | |
| "learning_rate": 4.933838450696757e-06, | |
| "logits/chosen": -2.9817254543304443, | |
| "logits/rejected": -3.0018677711486816, | |
| "logps/chosen": -0.8987566828727722, | |
| "logps/rejected": -1.0799884796142578, | |
| "loss": 0.9707, | |
| "odds_ratio_loss": 0.7198494672775269, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.08987566828727722, | |
| "rewards/margins": 0.018123187124729156, | |
| "rewards/rejected": -0.10799884796142578, | |
| "sft_loss": 0.8987566828727722, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.23708721422523285, | |
| "grad_norm": 3.667508840560913, | |
| "learning_rate": 4.923314217160234e-06, | |
| "logits/chosen": -2.988133192062378, | |
| "logits/rejected": -3.033163070678711, | |
| "logps/chosen": -0.9288301467895508, | |
| "logps/rejected": -1.1756031513214111, | |
| "loss": 0.9973, | |
| "odds_ratio_loss": 0.6846222877502441, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.09288302063941956, | |
| "rewards/margins": 0.024677302688360214, | |
| "rewards/rejected": -0.11756031215190887, | |
| "sft_loss": 0.9288301467895508, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.2540220152413209, | |
| "grad_norm": 1.4715845584869385, | |
| "learning_rate": 4.9120264984383285e-06, | |
| "logits/chosen": -2.997676134109497, | |
| "logits/rejected": -2.9935667514801025, | |
| "logps/chosen": -0.8239797353744507, | |
| "logps/rejected": -0.9983611106872559, | |
| "loss": 0.8869, | |
| "odds_ratio_loss": 0.6288361549377441, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.08239796757698059, | |
| "rewards/margins": 0.017438137903809547, | |
| "rewards/rejected": -0.09983611106872559, | |
| "sft_loss": 0.8239797353744507, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.270956816257409, | |
| "grad_norm": 1.3789633512496948, | |
| "learning_rate": 4.899978850820176e-06, | |
| "logits/chosen": -3.004080295562744, | |
| "logits/rejected": -3.0049595832824707, | |
| "logps/chosen": -0.9372480511665344, | |
| "logps/rejected": -1.147570252418518, | |
| "loss": 1.0008, | |
| "odds_ratio_loss": 0.635485053062439, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.0937248021364212, | |
| "rewards/margins": 0.021032210439443588, | |
| "rewards/rejected": -0.11475701630115509, | |
| "sft_loss": 0.9372480511665344, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.28789161727349705, | |
| "grad_norm": 2.1128056049346924, | |
| "learning_rate": 4.887175070016795e-06, | |
| "logits/chosen": -3.027259349822998, | |
| "logits/rejected": -3.0453531742095947, | |
| "logps/chosen": -0.7962668538093567, | |
| "logps/rejected": -1.0727083683013916, | |
| "loss": 0.8585, | |
| "odds_ratio_loss": 0.6222477555274963, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.07962668687105179, | |
| "rewards/margins": 0.027644142508506775, | |
| "rewards/rejected": -0.10727082192897797, | |
| "sft_loss": 0.7962668538093567, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.3048264182895851, | |
| "grad_norm": 4.011683464050293, | |
| "learning_rate": 4.873619189965217e-06, | |
| "logits/chosen": -2.973634719848633, | |
| "logits/rejected": -2.989631414413452, | |
| "logps/chosen": -0.8350585103034973, | |
| "logps/rejected": -1.051026463508606, | |
| "loss": 0.9, | |
| "odds_ratio_loss": 0.6493778824806213, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.08350586146116257, | |
| "rewards/margins": 0.021596785634756088, | |
| "rewards/rejected": -0.10510264337062836, | |
| "sft_loss": 0.8350585103034973, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.32176121930567314, | |
| "grad_norm": 1.2796882390975952, | |
| "learning_rate": 4.859315481557563e-06, | |
| "logits/chosen": -2.9721431732177734, | |
| "logits/rejected": -2.9849448204040527, | |
| "logps/chosen": -0.821880042552948, | |
| "logps/rejected": -1.0275905132293701, | |
| "loss": 0.8867, | |
| "odds_ratio_loss": 0.648421585559845, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.08218801021575928, | |
| "rewards/margins": 0.0205710306763649, | |
| "rewards/rejected": -0.10275904089212418, | |
| "sft_loss": 0.821880042552948, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.3386960203217612, | |
| "grad_norm": 1.8488657474517822, | |
| "learning_rate": 4.84426845129546e-06, | |
| "logits/chosen": -3.0222136974334717, | |
| "logits/rejected": -3.031564235687256, | |
| "logps/chosen": -0.8574220538139343, | |
| "logps/rejected": -0.9888635873794556, | |
| "loss": 0.9247, | |
| "odds_ratio_loss": 0.6725481748580933, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.08574221283197403, | |
| "rewards/margins": 0.013144141063094139, | |
| "rewards/rejected": -0.09888636320829391, | |
| "sft_loss": 0.8574220538139343, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.3556308213378493, | |
| "grad_norm": 1.5179060697555542, | |
| "learning_rate": 4.828482839870233e-06, | |
| "logits/chosen": -3.010849714279175, | |
| "logits/rejected": -3.0288453102111816, | |
| "logps/chosen": -0.8511277437210083, | |
| "logps/rejected": -0.9780422449111938, | |
| "loss": 0.9215, | |
| "odds_ratio_loss": 0.7037326693534851, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08511276543140411, | |
| "rewards/margins": 0.012691453099250793, | |
| "rewards/rejected": -0.0978042259812355, | |
| "sft_loss": 0.8511277437210083, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.37256562235393736, | |
| "grad_norm": 1.2089393138885498, | |
| "learning_rate": 4.811963620669314e-06, | |
| "logits/chosen": -3.0581696033477783, | |
| "logits/rejected": -3.099684476852417, | |
| "logps/chosen": -0.8706866502761841, | |
| "logps/rejected": -1.0248219966888428, | |
| "loss": 0.9377, | |
| "odds_ratio_loss": 0.6702378988265991, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.08706866204738617, | |
| "rewards/margins": 0.015413539484143257, | |
| "rewards/rejected": -0.10248219966888428, | |
| "sft_loss": 0.8706866502761841, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.3895004233700254, | |
| "grad_norm": 2.3151497840881348, | |
| "learning_rate": 4.794715998209328e-06, | |
| "logits/chosen": -2.9359335899353027, | |
| "logits/rejected": -2.9507675170898438, | |
| "logps/chosen": -0.8457564115524292, | |
| "logps/rejected": -1.0168864727020264, | |
| "loss": 0.9116, | |
| "odds_ratio_loss": 0.6584563255310059, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.08457564562559128, | |
| "rewards/margins": 0.017113011330366135, | |
| "rewards/rejected": -0.10168864578008652, | |
| "sft_loss": 0.8457564115524292, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.40643522438611346, | |
| "grad_norm": 2.3710029125213623, | |
| "learning_rate": 4.7767454064963724e-06, | |
| "logits/chosen": -2.9894230365753174, | |
| "logits/rejected": -3.022892475128174, | |
| "logps/chosen": -0.8459660410881042, | |
| "logps/rejected": -1.015751838684082, | |
| "loss": 0.9112, | |
| "odds_ratio_loss": 0.6523627042770386, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.08459659665822983, | |
| "rewards/margins": 0.016978587955236435, | |
| "rewards/rejected": -0.10157518088817596, | |
| "sft_loss": 0.8459660410881042, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.42337002540220153, | |
| "grad_norm": 2.1464998722076416, | |
| "learning_rate": 4.758057507313987e-06, | |
| "logits/chosen": -3.024229049682617, | |
| "logits/rejected": -3.034865617752075, | |
| "logps/chosen": -0.8055688142776489, | |
| "logps/rejected": -1.001123070716858, | |
| "loss": 0.869, | |
| "odds_ratio_loss": 0.6340595483779907, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.08055686950683594, | |
| "rewards/margins": 0.019555436447262764, | |
| "rewards/rejected": -0.10011231899261475, | |
| "sft_loss": 0.8055688142776489, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.4403048264182896, | |
| "grad_norm": 3.3724582195281982, | |
| "learning_rate": 4.73865818843936e-06, | |
| "logits/chosen": -3.042086124420166, | |
| "logits/rejected": -3.0638880729675293, | |
| "logps/chosen": -0.912920355796814, | |
| "logps/rejected": -1.1694436073303223, | |
| "loss": 0.9795, | |
| "odds_ratio_loss": 0.6657846570014954, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.09129203855991364, | |
| "rewards/margins": 0.025652330368757248, | |
| "rewards/rejected": -0.11694437265396118, | |
| "sft_loss": 0.912920355796814, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.4572396274343776, | |
| "grad_norm": 1.6793408393859863, | |
| "learning_rate": 4.718553561788339e-06, | |
| "logits/chosen": -3.0312306880950928, | |
| "logits/rejected": -3.060295820236206, | |
| "logps/chosen": -0.836656928062439, | |
| "logps/rejected": -0.986899733543396, | |
| "loss": 0.9016, | |
| "odds_ratio_loss": 0.6491778492927551, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.08366570621728897, | |
| "rewards/margins": 0.015024276450276375, | |
| "rewards/rejected": -0.0986899808049202, | |
| "sft_loss": 0.836656928062439, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.4741744284504657, | |
| "grad_norm": 2.6274185180664062, | |
| "learning_rate": 4.697749961489822e-06, | |
| "logits/chosen": -3.061577558517456, | |
| "logits/rejected": -3.0702619552612305, | |
| "logps/chosen": -0.8848905563354492, | |
| "logps/rejected": -1.0645692348480225, | |
| "loss": 0.9531, | |
| "odds_ratio_loss": 0.6824666857719421, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.08848904818296432, | |
| "rewards/margins": 0.017967868596315384, | |
| "rewards/rejected": -0.10645692050457001, | |
| "sft_loss": 0.8848905563354492, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.4911092294665538, | |
| "grad_norm": 2.089768886566162, | |
| "learning_rate": 4.67625394189013e-06, | |
| "logits/chosen": -3.0883841514587402, | |
| "logits/rejected": -3.090940475463867, | |
| "logps/chosen": -0.7822630405426025, | |
| "logps/rejected": -1.0061594247817993, | |
| "loss": 0.8425, | |
| "odds_ratio_loss": 0.6027771830558777, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.07822629809379578, | |
| "rewards/margins": 0.022389648482203484, | |
| "rewards/rejected": -0.100615955889225, | |
| "sft_loss": 0.7822630405426025, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.5080440304826418, | |
| "grad_norm": 2.2402989864349365, | |
| "learning_rate": 4.654072275488016e-06, | |
| "logits/chosen": -3.1045384407043457, | |
| "logits/rejected": -3.1042561531066895, | |
| "logps/chosen": -0.7530822157859802, | |
| "logps/rejected": -0.9303094744682312, | |
| "loss": 0.8152, | |
| "odds_ratio_loss": 0.621616542339325, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.07530822604894638, | |
| "rewards/margins": 0.017722725868225098, | |
| "rewards/rejected": -0.09303095191717148, | |
| "sft_loss": 0.7530822157859802, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.5249788314987299, | |
| "grad_norm": 7.859750747680664, | |
| "learning_rate": 4.631211950800925e-06, | |
| "logits/chosen": -3.0885701179504395, | |
| "logits/rejected": -3.1314892768859863, | |
| "logps/chosen": -0.8193107843399048, | |
| "logps/rejected": -1.0114855766296387, | |
| "loss": 0.8855, | |
| "odds_ratio_loss": 0.6622810959815979, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.08193108439445496, | |
| "rewards/margins": 0.01921747997403145, | |
| "rewards/rejected": -0.1011485680937767, | |
| "sft_loss": 0.8193107843399048, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.541913632514818, | |
| "grad_norm": 3.1328227519989014, | |
| "learning_rate": 4.6076801701632095e-06, | |
| "logits/chosen": -3.087184190750122, | |
| "logits/rejected": -3.1436970233917236, | |
| "logps/chosen": -0.801365852355957, | |
| "logps/rejected": -0.9046095609664917, | |
| "loss": 0.8686, | |
| "odds_ratio_loss": 0.6719350814819336, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.08013658970594406, | |
| "rewards/margins": 0.010324367322027683, | |
| "rewards/rejected": -0.09046096354722977, | |
| "sft_loss": 0.801365852355957, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.558848433530906, | |
| "grad_norm": 2.4766950607299805, | |
| "learning_rate": 4.583484347456972e-06, | |
| "logits/chosen": -3.0817341804504395, | |
| "logits/rejected": -3.114269256591797, | |
| "logps/chosen": -0.8467103242874146, | |
| "logps/rejected": -0.9794108271598816, | |
| "loss": 0.9136, | |
| "odds_ratio_loss": 0.6689561009407043, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.0846710279583931, | |
| "rewards/margins": 0.013270048424601555, | |
| "rewards/rejected": -0.0979410782456398, | |
| "sft_loss": 0.8467103242874146, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.5757832345469941, | |
| "grad_norm": 1.4625582695007324, | |
| "learning_rate": 4.55863210577626e-06, | |
| "logits/chosen": -3.123152256011963, | |
| "logits/rejected": -3.1361846923828125, | |
| "logps/chosen": -0.8682034611701965, | |
| "logps/rejected": -1.090419888496399, | |
| "loss": 0.9349, | |
| "odds_ratio_loss": 0.6673767566680908, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.08682034909725189, | |
| "rewards/margins": 0.022221634164452553, | |
| "rewards/rejected": -0.10904198884963989, | |
| "sft_loss": 0.8682034611701965, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.5927180355630821, | |
| "grad_norm": 1.6330158710479736, | |
| "learning_rate": 4.5331312750253465e-06, | |
| "logits/chosen": -3.079641103744507, | |
| "logits/rejected": -3.1221771240234375, | |
| "logps/chosen": -0.7776955962181091, | |
| "logps/rejected": -0.978277325630188, | |
| "loss": 0.8441, | |
| "odds_ratio_loss": 0.6638715267181396, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.07776956260204315, | |
| "rewards/margins": 0.02005816623568535, | |
| "rewards/rejected": -0.0978277251124382, | |
| "sft_loss": 0.7776955962181091, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.6096528365791702, | |
| "grad_norm": 1.9444369077682495, | |
| "learning_rate": 4.506989889451858e-06, | |
| "logits/chosen": -3.1103367805480957, | |
| "logits/rejected": -3.1434414386749268, | |
| "logps/chosen": -0.8445068597793579, | |
| "logps/rejected": -0.9869591593742371, | |
| "loss": 0.9108, | |
| "odds_ratio_loss": 0.662727952003479, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.08445067703723907, | |
| "rewards/margins": 0.014245236292481422, | |
| "rewards/rejected": -0.09869591891765594, | |
| "sft_loss": 0.8445068597793579, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.6265876375952583, | |
| "grad_norm": 2.251150369644165, | |
| "learning_rate": 4.480216185115512e-06, | |
| "logits/chosen": -3.0998404026031494, | |
| "logits/rejected": -3.087653398513794, | |
| "logps/chosen": -0.8066253662109375, | |
| "logps/rejected": -0.9696222543716431, | |
| "loss": 0.8704, | |
| "odds_ratio_loss": 0.6379208564758301, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08066253364086151, | |
| "rewards/margins": 0.016299689188599586, | |
| "rewards/rejected": -0.09696222841739655, | |
| "sft_loss": 0.8066253662109375, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.6435224386113463, | |
| "grad_norm": 1.8241935968399048, | |
| "learning_rate": 4.4528185972932856e-06, | |
| "logits/chosen": -3.019221782684326, | |
| "logits/rejected": -3.083693265914917, | |
| "logps/chosen": -0.8054075241088867, | |
| "logps/rejected": -1.0394740104675293, | |
| "loss": 0.8728, | |
| "odds_ratio_loss": 0.6744040846824646, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08054076135158539, | |
| "rewards/margins": 0.023406637832522392, | |
| "rewards/rejected": -0.10394741594791412, | |
| "sft_loss": 0.8054075241088867, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.6604572396274344, | |
| "grad_norm": 7.428930759429932, | |
| "learning_rate": 4.424805757821803e-06, | |
| "logits/chosen": -3.0259501934051514, | |
| "logits/rejected": -3.0692214965820312, | |
| "logps/chosen": -0.8603572845458984, | |
| "logps/rejected": -0.9688261151313782, | |
| "loss": 0.9297, | |
| "odds_ratio_loss": 0.6930958032608032, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.08603573590517044, | |
| "rewards/margins": 0.010846875607967377, | |
| "rewards/rejected": -0.09688261151313782, | |
| "sft_loss": 0.8603572845458984, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.6773920406435224, | |
| "grad_norm": 1.7724531888961792, | |
| "learning_rate": 4.396186492377812e-06, | |
| "logits/chosen": -3.049534797668457, | |
| "logits/rejected": -3.0946240425109863, | |
| "logps/chosen": -0.804740309715271, | |
| "logps/rejected": -1.0287699699401855, | |
| "loss": 0.8638, | |
| "odds_ratio_loss": 0.5903798341751099, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.08047403395175934, | |
| "rewards/margins": 0.02240295708179474, | |
| "rewards/rejected": -0.10287699848413467, | |
| "sft_loss": 0.804740309715271, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.6943268416596104, | |
| "grad_norm": 2.2618014812469482, | |
| "learning_rate": 4.366969817697578e-06, | |
| "logits/chosen": -3.0235114097595215, | |
| "logits/rejected": -3.083573341369629, | |
| "logps/chosen": -0.7958801984786987, | |
| "logps/rejected": -0.9750314950942993, | |
| "loss": 0.8637, | |
| "odds_ratio_loss": 0.6784581542015076, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.07958801835775375, | |
| "rewards/margins": 0.01791512593626976, | |
| "rewards/rejected": -0.09750314056873322, | |
| "sft_loss": 0.7958801984786987, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.7112616426756986, | |
| "grad_norm": 2.504657506942749, | |
| "learning_rate": 4.337164938736086e-06, | |
| "logits/chosen": -3.049118757247925, | |
| "logits/rejected": -3.05595326423645, | |
| "logps/chosen": -0.8237883448600769, | |
| "logps/rejected": -0.918091893196106, | |
| "loss": 0.8939, | |
| "odds_ratio_loss": 0.7009984254837036, | |
| "rewards/accuracies": 0.4749999940395355, | |
| "rewards/chosen": -0.08237884193658829, | |
| "rewards/margins": 0.00943034328520298, | |
| "rewards/rejected": -0.09180918335914612, | |
| "sft_loss": 0.8237883448600769, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.7281964436917866, | |
| "grad_norm": 1.2601932287216187, | |
| "learning_rate": 4.306781245766945e-06, | |
| "logits/chosen": -3.0380876064300537, | |
| "logits/rejected": -3.0915472507476807, | |
| "logps/chosen": -0.7913134098052979, | |
| "logps/rejected": -1.0263410806655884, | |
| "loss": 0.8569, | |
| "odds_ratio_loss": 0.6553653478622437, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.0791313424706459, | |
| "rewards/margins": 0.023502767086029053, | |
| "rewards/rejected": -0.10263410955667496, | |
| "sft_loss": 0.7913134098052979, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.7451312447078747, | |
| "grad_norm": 4.184472560882568, | |
| "learning_rate": 4.275828311423903e-06, | |
| "logits/chosen": -3.093045473098755, | |
| "logits/rejected": -3.094749927520752, | |
| "logps/chosen": -0.8900951147079468, | |
| "logps/rejected": -0.9482099413871765, | |
| "loss": 0.9611, | |
| "odds_ratio_loss": 0.7105128765106201, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08900952339172363, | |
| "rewards/margins": 0.005811482667922974, | |
| "rewards/rejected": -0.09482099860906601, | |
| "sft_loss": 0.8900951147079468, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.7620660457239627, | |
| "grad_norm": 5.296163082122803, | |
| "learning_rate": 4.244315887684912e-06, | |
| "logits/chosen": -3.063938617706299, | |
| "logits/rejected": -3.0689697265625, | |
| "logps/chosen": -0.7619583010673523, | |
| "logps/rejected": -0.9321343302726746, | |
| "loss": 0.8277, | |
| "odds_ratio_loss": 0.6576007008552551, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.07619582116603851, | |
| "rewards/margins": 0.017017606645822525, | |
| "rewards/rejected": -0.09321344643831253, | |
| "sft_loss": 0.7619583010673523, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.7790008467400508, | |
| "grad_norm": 1.0935202836990356, | |
| "learning_rate": 4.212253902799685e-06, | |
| "logits/chosen": -3.107138156890869, | |
| "logits/rejected": -3.1228718757629395, | |
| "logps/chosen": -0.828906238079071, | |
| "logps/rejected": -1.0854886770248413, | |
| "loss": 0.8936, | |
| "odds_ratio_loss": 0.6471681594848633, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.08289062976837158, | |
| "rewards/margins": 0.02565823495388031, | |
| "rewards/rejected": -0.10854886472225189, | |
| "sft_loss": 0.828906238079071, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.7959356477561389, | |
| "grad_norm": 2.112194538116455, | |
| "learning_rate": 4.179652458161718e-06, | |
| "logits/chosen": -3.0884511470794678, | |
| "logits/rejected": -3.086930751800537, | |
| "logps/chosen": -0.8770621418952942, | |
| "logps/rejected": -0.9670013189315796, | |
| "loss": 0.9476, | |
| "odds_ratio_loss": 0.7056951522827148, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.08770622313022614, | |
| "rewards/margins": 0.008993919007480145, | |
| "rewards/rejected": -0.09670013934373856, | |
| "sft_loss": 0.8770621418952942, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.8128704487722269, | |
| "grad_norm": 1.4803324937820435, | |
| "learning_rate": 4.146521825125765e-06, | |
| "logits/chosen": -3.1123909950256348, | |
| "logits/rejected": -3.127878189086914, | |
| "logps/chosen": -0.8552261590957642, | |
| "logps/rejected": -0.9841393232345581, | |
| "loss": 0.9226, | |
| "odds_ratio_loss": 0.6738197803497314, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.0855226144194603, | |
| "rewards/margins": 0.012891319580376148, | |
| "rewards/rejected": -0.09841393679380417, | |
| "sft_loss": 0.8552261590957642, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.8298052497883149, | |
| "grad_norm": 1.639560580253601, | |
| "learning_rate": 4.11287244177176e-06, | |
| "logits/chosen": -3.1404106616973877, | |
| "logits/rejected": -3.1268608570098877, | |
| "logps/chosen": -0.7970486283302307, | |
| "logps/rejected": -1.0359351634979248, | |
| "loss": 0.8576, | |
| "odds_ratio_loss": 0.6055651307106018, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.07970486581325531, | |
| "rewards/margins": 0.02388865128159523, | |
| "rewards/rejected": -0.10359351336956024, | |
| "sft_loss": 0.7970486283302307, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.8467400508044031, | |
| "grad_norm": 7.098996162414551, | |
| "learning_rate": 4.078714909616215e-06, | |
| "logits/chosen": -3.1334495544433594, | |
| "logits/rejected": -3.127417802810669, | |
| "logps/chosen": -0.8159440755844116, | |
| "logps/rejected": -1.052442193031311, | |
| "loss": 0.8758, | |
| "odds_ratio_loss": 0.5985492467880249, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.08159441500902176, | |
| "rewards/margins": 0.02364981174468994, | |
| "rewards/rejected": -0.10524420440196991, | |
| "sft_loss": 0.8159440755844116, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.8467400508044031, | |
| "eval_logits/chosen": -3.1088695526123047, | |
| "eval_logits/rejected": -3.1276028156280518, | |
| "eval_logps/chosen": -0.8053962588310242, | |
| "eval_logps/rejected": -1.0085769891738892, | |
| "eval_loss": 0.8691067099571228, | |
| "eval_odds_ratio_loss": 0.6371051073074341, | |
| "eval_rewards/accuracies": 0.5704761743545532, | |
| "eval_rewards/chosen": -0.08053962886333466, | |
| "eval_rewards/margins": 0.02031807415187359, | |
| "eval_rewards/rejected": -0.1008576974272728, | |
| "eval_runtime": 194.1916, | |
| "eval_samples_per_second": 5.407, | |
| "eval_sft_loss": 0.8053962588310242, | |
| "eval_steps_per_second": 2.704, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.8636748518204911, | |
| "grad_norm": 5.1792216300964355, | |
| "learning_rate": 4.044059990272125e-06, | |
| "logits/chosen": -3.1320395469665527, | |
| "logits/rejected": -3.1584267616271973, | |
| "logps/chosen": -0.8421246409416199, | |
| "logps/rejected": -1.076226830482483, | |
| "loss": 0.9067, | |
| "odds_ratio_loss": 0.64554762840271, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08421246707439423, | |
| "rewards/margins": 0.02341020107269287, | |
| "rewards/rejected": -0.10762268304824829, | |
| "sft_loss": 0.8421246409416199, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 0.8806096528365792, | |
| "grad_norm": 2.52073073387146, | |
| "learning_rate": 4.0089186020584345e-06, | |
| "logits/chosen": -3.1114721298217773, | |
| "logits/rejected": -3.1454081535339355, | |
| "logps/chosen": -0.8953489065170288, | |
| "logps/rejected": -1.0218431949615479, | |
| "loss": 0.9597, | |
| "odds_ratio_loss": 0.6433413028717041, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.08953489363193512, | |
| "rewards/margins": 0.012649421580135822, | |
| "rewards/rejected": -0.10218431800603867, | |
| "sft_loss": 0.8953489065170288, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 0.8975444538526672, | |
| "grad_norm": 3.2529776096343994, | |
| "learning_rate": 3.973301816560124e-06, | |
| "logits/chosen": -3.1164424419403076, | |
| "logits/rejected": -3.0959460735321045, | |
| "logps/chosen": -0.8098430633544922, | |
| "logps/rejected": -1.0107625722885132, | |
| "loss": 0.871, | |
| "odds_ratio_loss": 0.6116595268249512, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.08098430931568146, | |
| "rewards/margins": 0.0200919471681118, | |
| "rewards/rejected": -0.10107626020908356, | |
| "sft_loss": 0.8098430633544922, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 0.9144792548687553, | |
| "grad_norm": 2.4143612384796143, | |
| "learning_rate": 3.937220855140021e-06, | |
| "logits/chosen": -3.1310832500457764, | |
| "logits/rejected": -3.1453144550323486, | |
| "logps/chosen": -0.8188160061836243, | |
| "logps/rejected": -0.9051122665405273, | |
| "loss": 0.8891, | |
| "odds_ratio_loss": 0.7029509544372559, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.08188159763813019, | |
| "rewards/margins": 0.008629636839032173, | |
| "rewards/rejected": -0.09051123261451721, | |
| "sft_loss": 0.8188160061836243, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 0.9314140558848434, | |
| "grad_norm": 1.3889896869659424, | |
| "learning_rate": 3.900687085403418e-06, | |
| "logits/chosen": -3.1416361331939697, | |
| "logits/rejected": -3.169236421585083, | |
| "logps/chosen": -0.8118529319763184, | |
| "logps/rejected": -0.8874411582946777, | |
| "loss": 0.8817, | |
| "odds_ratio_loss": 0.6982238292694092, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.08118529617786407, | |
| "rewards/margins": 0.007558824960142374, | |
| "rewards/rejected": -0.08874412626028061, | |
| "sft_loss": 0.8118529319763184, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.9483488569009314, | |
| "grad_norm": 2.055143356323242, | |
| "learning_rate": 3.863712017616614e-06, | |
| "logits/chosen": -3.131472110748291, | |
| "logits/rejected": -3.160679340362549, | |
| "logps/chosen": -0.8101698160171509, | |
| "logps/rejected": -0.9761570692062378, | |
| "loss": 0.8764, | |
| "odds_ratio_loss": 0.6618725657463074, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.08101697266101837, | |
| "rewards/margins": 0.016598742455244064, | |
| "rewards/rejected": -0.09761571884155273, | |
| "sft_loss": 0.8101698160171509, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 0.9652836579170194, | |
| "grad_norm": 14.660100936889648, | |
| "learning_rate": 3.826307301080504e-06, | |
| "logits/chosen": -3.0291128158569336, | |
| "logits/rejected": -3.0481247901916504, | |
| "logps/chosen": -0.8100768327713013, | |
| "logps/rejected": -1.162418007850647, | |
| "loss": 0.8764, | |
| "odds_ratio_loss": 0.6627860069274902, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.08100768178701401, | |
| "rewards/margins": 0.03523411601781845, | |
| "rewards/rejected": -0.11624179035425186, | |
| "sft_loss": 0.8100768327713013, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 0.9822184589331076, | |
| "grad_norm": 2.563514471054077, | |
| "learning_rate": 3.7884847204603775e-06, | |
| "logits/chosen": -3.1155200004577637, | |
| "logits/rejected": -3.15079665184021, | |
| "logps/chosen": -0.8116699457168579, | |
| "logps/rejected": -0.9339970350265503, | |
| "loss": 0.8812, | |
| "odds_ratio_loss": 0.6949950456619263, | |
| "rewards/accuracies": 0.48750001192092896, | |
| "rewards/chosen": -0.08116699010133743, | |
| "rewards/margins": 0.012232715263962746, | |
| "rewards/rejected": -0.09339970350265503, | |
| "sft_loss": 0.8116699457168579, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 0.9991532599491956, | |
| "grad_norm": 2.7657859325408936, | |
| "learning_rate": 3.750256192073058e-06, | |
| "logits/chosen": -3.172693967819214, | |
| "logits/rejected": -3.1799886226654053, | |
| "logps/chosen": -0.9269296526908875, | |
| "logps/rejected": -0.9760646820068359, | |
| "loss": 1.0009, | |
| "odds_ratio_loss": 0.7394477128982544, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.09269297122955322, | |
| "rewards/margins": 0.004913498647511005, | |
| "rewards/rejected": -0.09760646522045135, | |
| "sft_loss": 0.9269296526908875, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 1.0160880609652836, | |
| "grad_norm": 7.485799789428711, | |
| "learning_rate": 3.7116337601325715e-06, | |
| "logits/chosen": -3.1055843830108643, | |
| "logits/rejected": -3.129669189453125, | |
| "logps/chosen": -0.7523837089538574, | |
| "logps/rejected": -0.8988542556762695, | |
| "loss": 0.8166, | |
| "odds_ratio_loss": 0.6426426768302917, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.07523836940526962, | |
| "rewards/margins": 0.014647054485976696, | |
| "rewards/rejected": -0.08988542854785919, | |
| "sft_loss": 0.7523837089538574, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 1.0330228619813717, | |
| "grad_norm": 2.2944157123565674, | |
| "learning_rate": 3.6726295929555154e-06, | |
| "logits/chosen": -3.077573776245117, | |
| "logits/rejected": -3.11928391456604, | |
| "logps/chosen": -0.7445582747459412, | |
| "logps/rejected": -0.9014002680778503, | |
| "loss": 0.8117, | |
| "odds_ratio_loss": 0.6712278723716736, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.07445583492517471, | |
| "rewards/margins": 0.015684202313423157, | |
| "rewards/rejected": -0.09014002978801727, | |
| "sft_loss": 0.7445582747459412, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 1.0499576629974599, | |
| "grad_norm": 2.4759469032287598, | |
| "learning_rate": 3.6332559791273307e-06, | |
| "logits/chosen": -3.070753812789917, | |
| "logits/rejected": -3.133881092071533, | |
| "logps/chosen": -0.7787492871284485, | |
| "logps/rejected": -0.9435569643974304, | |
| "loss": 0.843, | |
| "odds_ratio_loss": 0.6423131823539734, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.07787492126226425, | |
| "rewards/margins": 0.016480756923556328, | |
| "rewards/rejected": -0.09435568749904633, | |
| "sft_loss": 0.7787492871284485, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 1.0668924640135478, | |
| "grad_norm": 2.397857666015625, | |
| "learning_rate": 3.593525323630681e-06, | |
| "logits/chosen": -3.0695629119873047, | |
| "logits/rejected": -3.0961527824401855, | |
| "logps/chosen": -0.8072765469551086, | |
| "logps/rejected": -0.9367998838424683, | |
| "loss": 0.8715, | |
| "odds_ratio_loss": 0.6417396664619446, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.08072765171527863, | |
| "rewards/margins": 0.012952342629432678, | |
| "rewards/rejected": -0.0936800017952919, | |
| "sft_loss": 0.8072765469551086, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 1.083827265029636, | |
| "grad_norm": 2.2752127647399902, | |
| "learning_rate": 3.5534501439371615e-06, | |
| "logits/chosen": -3.1115057468414307, | |
| "logits/rejected": -3.1277289390563965, | |
| "logps/chosen": -0.7920553088188171, | |
| "logps/rejected": -0.9753093719482422, | |
| "loss": 0.8548, | |
| "odds_ratio_loss": 0.6271573901176453, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.07920553535223007, | |
| "rewards/margins": 0.018325407058000565, | |
| "rewards/rejected": -0.09753094613552094, | |
| "sft_loss": 0.7920553088188171, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 1.100762066045724, | |
| "grad_norm": 1.311848521232605, | |
| "learning_rate": 3.5130430660635633e-06, | |
| "logits/chosen": -3.1020348072052, | |
| "logits/rejected": -3.1430366039276123, | |
| "logps/chosen": -0.8032411336898804, | |
| "logps/rejected": -0.9853434562683105, | |
| "loss": 0.864, | |
| "odds_ratio_loss": 0.6071646809577942, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.08032412827014923, | |
| "rewards/margins": 0.018210221081972122, | |
| "rewards/rejected": -0.09853433817625046, | |
| "sft_loss": 0.8032411336898804, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 1.117696867061812, | |
| "grad_norm": 1.6412379741668701, | |
| "learning_rate": 3.4723168205939444e-06, | |
| "logits/chosen": -3.0930168628692627, | |
| "logits/rejected": -3.1322126388549805, | |
| "logps/chosen": -0.7815112471580505, | |
| "logps/rejected": -0.9042918086051941, | |
| "loss": 0.8483, | |
| "odds_ratio_loss": 0.6679799556732178, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.07815112918615341, | |
| "rewards/margins": 0.012278061360120773, | |
| "rewards/rejected": -0.09042918682098389, | |
| "sft_loss": 0.7815112471580505, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 1.1346316680779, | |
| "grad_norm": 2.1591804027557373, | |
| "learning_rate": 3.431284238668754e-06, | |
| "logits/chosen": -3.062398910522461, | |
| "logits/rejected": -3.1151247024536133, | |
| "logps/chosen": -0.8405235409736633, | |
| "logps/rejected": -1.0043154954910278, | |
| "loss": 0.9059, | |
| "odds_ratio_loss": 0.6538293957710266, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08405234664678574, | |
| "rewards/margins": 0.01637919992208481, | |
| "rewards/rejected": -0.10043156147003174, | |
| "sft_loss": 0.8405235409736633, | |
| "step": 670 | |
| }, | |
| { | |
| "epoch": 1.1515664690939882, | |
| "grad_norm": 4.811310291290283, | |
| "learning_rate": 3.389958247942274e-06, | |
| "logits/chosen": -3.057506561279297, | |
| "logits/rejected": -3.1269962787628174, | |
| "logps/chosen": -0.8411120176315308, | |
| "logps/rejected": -1.1102626323699951, | |
| "loss": 0.9091, | |
| "odds_ratio_loss": 0.6803519129753113, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.08411119878292084, | |
| "rewards/margins": 0.026915064081549644, | |
| "rewards/rejected": -0.11102626472711563, | |
| "sft_loss": 0.8411120176315308, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 1.168501270110076, | |
| "grad_norm": 2.754120111465454, | |
| "learning_rate": 3.3483518685096588e-06, | |
| "logits/chosen": -3.08880615234375, | |
| "logits/rejected": -3.109083890914917, | |
| "logps/chosen": -0.8396803140640259, | |
| "logps/rejected": -0.9775651693344116, | |
| "loss": 0.9066, | |
| "odds_ratio_loss": 0.6687448620796204, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.08396803587675095, | |
| "rewards/margins": 0.013788496144115925, | |
| "rewards/rejected": -0.097756527364254, | |
| "sft_loss": 0.8396803140640259, | |
| "step": 690 | |
| }, | |
| { | |
| "epoch": 1.1854360711261642, | |
| "grad_norm": 1.9642765522003174, | |
| "learning_rate": 3.306478208804839e-06, | |
| "logits/chosen": -3.0659680366516113, | |
| "logits/rejected": -3.1026992797851562, | |
| "logps/chosen": -0.7790535092353821, | |
| "logps/rejected": -0.9482936859130859, | |
| "loss": 0.8451, | |
| "odds_ratio_loss": 0.659988284111023, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.07790535688400269, | |
| "rewards/margins": 0.016924021765589714, | |
| "rewards/rejected": -0.09482936561107635, | |
| "sft_loss": 0.7790535092353821, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 1.2023708721422524, | |
| "grad_norm": 2.6194725036621094, | |
| "learning_rate": 3.264350461470608e-06, | |
| "logits/chosen": -3.0373668670654297, | |
| "logits/rejected": -3.0688533782958984, | |
| "logps/chosen": -0.7216525077819824, | |
| "logps/rejected": -1.0178143978118896, | |
| "loss": 0.7814, | |
| "odds_ratio_loss": 0.597113311290741, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.07216525077819824, | |
| "rewards/margins": 0.02961619198322296, | |
| "rewards/rejected": -0.1017814427614212, | |
| "sft_loss": 0.7216525077819824, | |
| "step": 710 | |
| }, | |
| { | |
| "epoch": 1.2193056731583405, | |
| "grad_norm": 2.312389373779297, | |
| "learning_rate": 3.2219818992021685e-06, | |
| "logits/chosen": -3.03488826751709, | |
| "logits/rejected": -3.087043285369873, | |
| "logps/chosen": -0.7246071696281433, | |
| "logps/rejected": -1.047911286354065, | |
| "loss": 0.7828, | |
| "odds_ratio_loss": 0.5823417901992798, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.07246071845293045, | |
| "rewards/margins": 0.03233041241765022, | |
| "rewards/rejected": -0.10479112714529037, | |
| "sft_loss": 0.7246071696281433, | |
| "step": 720 | |
| }, | |
| { | |
| "epoch": 1.2362404741744284, | |
| "grad_norm": 7.234289169311523, | |
| "learning_rate": 3.1793858705654595e-06, | |
| "logits/chosen": -3.0948994159698486, | |
| "logits/rejected": -3.1080453395843506, | |
| "logps/chosen": -0.7130419611930847, | |
| "logps/rejected": -0.9181682467460632, | |
| "loss": 0.7733, | |
| "odds_ratio_loss": 0.6023129820823669, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.07130420207977295, | |
| "rewards/margins": 0.020512625575065613, | |
| "rewards/rejected": -0.09181682765483856, | |
| "sft_loss": 0.7130419611930847, | |
| "step": 730 | |
| }, | |
| { | |
| "epoch": 1.2531752751905165, | |
| "grad_norm": 2.101661443710327, | |
| "learning_rate": 3.1365757957915787e-06, | |
| "logits/chosen": -3.0874876976013184, | |
| "logits/rejected": -3.123832941055298, | |
| "logps/chosen": -0.8178297877311707, | |
| "logps/rejected": -0.9593319892883301, | |
| "loss": 0.8832, | |
| "odds_ratio_loss": 0.6534532308578491, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.0817829817533493, | |
| "rewards/margins": 0.01415021438151598, | |
| "rewards/rejected": -0.09593319892883301, | |
| "sft_loss": 0.8178297877311707, | |
| "step": 740 | |
| }, | |
| { | |
| "epoch": 1.2701100762066047, | |
| "grad_norm": 3.2394182682037354, | |
| "learning_rate": 3.093565162548633e-06, | |
| "logits/chosen": -3.0456290245056152, | |
| "logits/rejected": -3.0783424377441406, | |
| "logps/chosen": -0.8610566854476929, | |
| "logps/rejected": -1.0578378438949585, | |
| "loss": 0.9282, | |
| "odds_ratio_loss": 0.6714938879013062, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08610567450523376, | |
| "rewards/margins": 0.019678115844726562, | |
| "rewards/rejected": -0.10578378289937973, | |
| "sft_loss": 0.8610566854476929, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 1.2870448772226926, | |
| "grad_norm": 3.4462602138519287, | |
| "learning_rate": 3.0503675216923294e-06, | |
| "logits/chosen": -3.112204074859619, | |
| "logits/rejected": -3.1115825176239014, | |
| "logps/chosen": -0.7481369376182556, | |
| "logps/rejected": -0.9443623423576355, | |
| "loss": 0.8093, | |
| "odds_ratio_loss": 0.6117558479309082, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.07481369376182556, | |
| "rewards/margins": 0.019622545689344406, | |
| "rewards/rejected": -0.09443624317646027, | |
| "sft_loss": 0.7481369376182556, | |
| "step": 760 | |
| }, | |
| { | |
| "epoch": 1.3039796782387807, | |
| "grad_norm": 2.5497653484344482, | |
| "learning_rate": 3.0069964829966748e-06, | |
| "logits/chosen": -3.083761692047119, | |
| "logits/rejected": -3.1253674030303955, | |
| "logps/chosen": -0.7777606248855591, | |
| "logps/rejected": -0.9011049270629883, | |
| "loss": 0.844, | |
| "odds_ratio_loss": 0.6621277928352356, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.07777605950832367, | |
| "rewards/margins": 0.01233443059027195, | |
| "rewards/rejected": -0.09011048078536987, | |
| "sft_loss": 0.7777606248855591, | |
| "step": 770 | |
| }, | |
| { | |
| "epoch": 1.3209144792548688, | |
| "grad_norm": 2.1115994453430176, | |
| "learning_rate": 2.963465710866094e-06, | |
| "logits/chosen": -3.098053455352783, | |
| "logits/rejected": -3.1173479557037354, | |
| "logps/chosen": -0.7589127421379089, | |
| "logps/rejected": -1.0739343166351318, | |
| "loss": 0.8183, | |
| "odds_ratio_loss": 0.5940018892288208, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.07589127868413925, | |
| "rewards/margins": 0.03150214999914169, | |
| "rewards/rejected": -0.10739342123270035, | |
| "sft_loss": 0.7589127421379089, | |
| "step": 780 | |
| }, | |
| { | |
| "epoch": 1.337849280270957, | |
| "grad_norm": 7.320650577545166, | |
| "learning_rate": 2.919788920030357e-06, | |
| "logits/chosen": -3.143812894821167, | |
| "logits/rejected": -3.1576976776123047, | |
| "logps/chosen": -0.8158448934555054, | |
| "logps/rejected": -0.9654959440231323, | |
| "loss": 0.8826, | |
| "odds_ratio_loss": 0.6670835614204407, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.08158449828624725, | |
| "rewards/margins": 0.014965096488595009, | |
| "rewards/rejected": -0.09654959291219711, | |
| "sft_loss": 0.8158448934555054, | |
| "step": 790 | |
| }, | |
| { | |
| "epoch": 1.3547840812870449, | |
| "grad_norm": 1.3826831579208374, | |
| "learning_rate": 2.8759798712236303e-06, | |
| "logits/chosen": -3.1377153396606445, | |
| "logits/rejected": -3.138549566268921, | |
| "logps/chosen": -0.7528073191642761, | |
| "logps/rejected": -1.0746774673461914, | |
| "loss": 0.8173, | |
| "odds_ratio_loss": 0.6448089480400085, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.07528072595596313, | |
| "rewards/margins": 0.032187022268772125, | |
| "rewards/rejected": -0.10746775567531586, | |
| "sft_loss": 0.7528073191642761, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 1.371718882303133, | |
| "grad_norm": 2.1327664852142334, | |
| "learning_rate": 2.8320523668490507e-06, | |
| "logits/chosen": -3.095376968383789, | |
| "logits/rejected": -3.129220485687256, | |
| "logps/chosen": -0.8166864514350891, | |
| "logps/rejected": -0.9957429766654968, | |
| "loss": 0.8841, | |
| "odds_ratio_loss": 0.6738199591636658, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08166865259408951, | |
| "rewards/margins": 0.01790565252304077, | |
| "rewards/rejected": -0.09957430511713028, | |
| "sft_loss": 0.8166864514350891, | |
| "step": 810 | |
| }, | |
| { | |
| "epoch": 1.388653683319221, | |
| "grad_norm": 2.5101046562194824, | |
| "learning_rate": 2.7880202466301597e-06, | |
| "logits/chosen": -3.075023651123047, | |
| "logits/rejected": -3.1098039150238037, | |
| "logps/chosen": -0.8058909177780151, | |
| "logps/rejected": -0.9401804208755493, | |
| "loss": 0.8735, | |
| "odds_ratio_loss": 0.6758350133895874, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.08058909326791763, | |
| "rewards/margins": 0.013428950682282448, | |
| "rewards/rejected": -0.09401804953813553, | |
| "sft_loss": 0.8058909177780151, | |
| "step": 820 | |
| }, | |
| { | |
| "epoch": 1.405588484335309, | |
| "grad_norm": 2.202185869216919, | |
| "learning_rate": 2.7438973832505854e-06, | |
| "logits/chosen": -3.060824155807495, | |
| "logits/rejected": -3.0545971393585205, | |
| "logps/chosen": -0.7594717741012573, | |
| "logps/rejected": -0.9759091138839722, | |
| "loss": 0.8235, | |
| "odds_ratio_loss": 0.6401799321174622, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.07594718039035797, | |
| "rewards/margins": 0.021643735468387604, | |
| "rewards/rejected": -0.09759090840816498, | |
| "sft_loss": 0.7594717741012573, | |
| "step": 830 | |
| }, | |
| { | |
| "epoch": 1.4225232853513972, | |
| "grad_norm": 5.100254058837891, | |
| "learning_rate": 2.699697677983341e-06, | |
| "logits/chosen": -3.1682240962982178, | |
| "logits/rejected": -3.182861804962158, | |
| "logps/chosen": -0.8106738924980164, | |
| "logps/rejected": -0.8703418970108032, | |
| "loss": 0.8831, | |
| "odds_ratio_loss": 0.7241007685661316, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.08106739819049835, | |
| "rewards/margins": 0.0059667956084012985, | |
| "rewards/rejected": -0.0870341882109642, | |
| "sft_loss": 0.8106738924980164, | |
| "step": 840 | |
| }, | |
| { | |
| "epoch": 1.4394580863674853, | |
| "grad_norm": 7.667366981506348, | |
| "learning_rate": 2.6554350563111115e-06, | |
| "logits/chosen": -3.0851314067840576, | |
| "logits/rejected": -3.116276502609253, | |
| "logps/chosen": -0.8307350873947144, | |
| "logps/rejected": -0.9147516489028931, | |
| "loss": 0.9054, | |
| "odds_ratio_loss": 0.7461589574813843, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.08307350426912308, | |
| "rewards/margins": 0.0084016602486372, | |
| "rewards/rejected": -0.09147517383098602, | |
| "sft_loss": 0.8307350873947144, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 1.4563928873835732, | |
| "grad_norm": 6.640262126922607, | |
| "learning_rate": 2.611123463538913e-06, | |
| "logits/chosen": -3.0749902725219727, | |
| "logits/rejected": -3.0911591053009033, | |
| "logps/chosen": -0.7142345309257507, | |
| "logps/rejected": -0.9272225499153137, | |
| "loss": 0.7775, | |
| "odds_ratio_loss": 0.6324297189712524, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.07142344862222672, | |
| "rewards/margins": 0.02129879966378212, | |
| "rewards/rejected": -0.09272225201129913, | |
| "sft_loss": 0.7142345309257507, | |
| "step": 860 | |
| }, | |
| { | |
| "epoch": 1.4733276883996613, | |
| "grad_norm": 8.2167329788208, | |
| "learning_rate": 2.566776860400514e-06, | |
| "logits/chosen": -3.1082987785339355, | |
| "logits/rejected": -3.1257712841033936, | |
| "logps/chosen": -0.8350755572319031, | |
| "logps/rejected": -1.056910753250122, | |
| "loss": 0.898, | |
| "odds_ratio_loss": 0.6290403604507446, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.08350756019353867, | |
| "rewards/margins": 0.022183528169989586, | |
| "rewards/rejected": -0.10569107532501221, | |
| "sft_loss": 0.8350755572319031, | |
| "step": 870 | |
| }, | |
| { | |
| "epoch": 1.4902624894157492, | |
| "grad_norm": 1.4808557033538818, | |
| "learning_rate": 2.522409218659989e-06, | |
| "logits/chosen": -3.143266201019287, | |
| "logits/rejected": -3.170633316040039, | |
| "logps/chosen": -0.7854605913162231, | |
| "logps/rejected": -0.9068530797958374, | |
| "loss": 0.8512, | |
| "odds_ratio_loss": 0.6573347449302673, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.07854606211185455, | |
| "rewards/margins": 0.012139257043600082, | |
| "rewards/rejected": -0.09068530797958374, | |
| "sft_loss": 0.7854605913162231, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 1.5071972904318374, | |
| "grad_norm": 3.6185755729675293, | |
| "learning_rate": 2.4780345167097976e-06, | |
| "logits/chosen": -3.0972650051116943, | |
| "logits/rejected": -3.0835044384002686, | |
| "logps/chosen": -0.7852433919906616, | |
| "logps/rejected": -1.0855656862258911, | |
| "loss": 0.8459, | |
| "odds_ratio_loss": 0.6065645813941956, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.07852433621883392, | |
| "rewards/margins": 0.030032237991690636, | |
| "rewards/rejected": -0.1085565835237503, | |
| "sft_loss": 0.7852433919906616, | |
| "step": 890 | |
| }, | |
| { | |
| "epoch": 1.5241320914479255, | |
| "grad_norm": 4.639472961425781, | |
| "learning_rate": 2.4336667351667747e-06, | |
| "logits/chosen": -3.114197015762329, | |
| "logits/rejected": -3.124145269393921, | |
| "logps/chosen": -0.8203206062316895, | |
| "logps/rejected": -1.0489810705184937, | |
| "loss": 0.8792, | |
| "odds_ratio_loss": 0.5887311697006226, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.08203206956386566, | |
| "rewards/margins": 0.022866051644086838, | |
| "rewards/rejected": -0.1048981174826622, | |
| "sft_loss": 0.8203206062316895, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 1.5410668924640136, | |
| "grad_norm": 3.153228282928467, | |
| "learning_rate": 2.3893198524674264e-06, | |
| "logits/chosen": -3.086516857147217, | |
| "logits/rejected": -3.104675769805908, | |
| "logps/chosen": -0.7846948504447937, | |
| "logps/rejected": -0.9825633764266968, | |
| "loss": 0.8478, | |
| "odds_ratio_loss": 0.6309666037559509, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.07846949249505997, | |
| "rewards/margins": 0.019786860793828964, | |
| "rewards/rejected": -0.09825634956359863, | |
| "sft_loss": 0.7846948504447937, | |
| "step": 910 | |
| }, | |
| { | |
| "epoch": 1.5580016934801018, | |
| "grad_norm": 3.532456874847412, | |
| "learning_rate": 2.345007840463904e-06, | |
| "logits/chosen": -3.0608856678009033, | |
| "logits/rejected": -3.0965914726257324, | |
| "logps/chosen": -0.8089407682418823, | |
| "logps/rejected": -0.9399329423904419, | |
| "loss": 0.8759, | |
| "odds_ratio_loss": 0.6692665815353394, | |
| "rewards/accuracies": 0.4937500059604645, | |
| "rewards/chosen": -0.08089407533407211, | |
| "rewards/margins": 0.013099217787384987, | |
| "rewards/rejected": -0.09399329125881195, | |
| "sft_loss": 0.8089407682418823, | |
| "step": 920 | |
| }, | |
| { | |
| "epoch": 1.5749364944961897, | |
| "grad_norm": 3.778723955154419, | |
| "learning_rate": 2.3007446600220572e-06, | |
| "logits/chosen": -3.122668504714966, | |
| "logits/rejected": -3.1052489280700684, | |
| "logps/chosen": -0.799084484577179, | |
| "logps/rejected": -0.9807415008544922, | |
| "loss": 0.8639, | |
| "odds_ratio_loss": 0.6483135223388672, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.07990844547748566, | |
| "rewards/margins": 0.018165703862905502, | |
| "rewards/rejected": -0.09807415306568146, | |
| "sft_loss": 0.799084484577179, | |
| "step": 930 | |
| }, | |
| { | |
| "epoch": 1.5918712955122776, | |
| "grad_norm": 3.475341558456421, | |
| "learning_rate": 2.2565442566229507e-06, | |
| "logits/chosen": -3.0740559101104736, | |
| "logits/rejected": -3.106633424758911, | |
| "logps/chosen": -0.7921947240829468, | |
| "logps/rejected": -0.8962046504020691, | |
| "loss": 0.862, | |
| "odds_ratio_loss": 0.6985131502151489, | |
| "rewards/accuracies": 0.4375, | |
| "rewards/chosen": -0.07921947538852692, | |
| "rewards/margins": 0.01040099561214447, | |
| "rewards/rejected": -0.08962046355009079, | |
| "sft_loss": 0.7921947240829468, | |
| "step": 940 | |
| }, | |
| { | |
| "epoch": 1.6088060965283657, | |
| "grad_norm": 2.1668930053710938, | |
| "learning_rate": 2.2124205559692195e-06, | |
| "logits/chosen": -3.064317226409912, | |
| "logits/rejected": -3.1167845726013184, | |
| "logps/chosen": -0.7817971706390381, | |
| "logps/rejected": -1.0080362558364868, | |
| "loss": 0.8399, | |
| "odds_ratio_loss": 0.5807359218597412, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.0781797245144844, | |
| "rewards/margins": 0.02262391336262226, | |
| "rewards/rejected": -0.10080362856388092, | |
| "sft_loss": 0.7817971706390381, | |
| "step": 950 | |
| }, | |
| { | |
| "epoch": 1.6257408975444538, | |
| "grad_norm": 1.8939071893692017, | |
| "learning_rate": 2.168387459597666e-06, | |
| "logits/chosen": -3.092590808868408, | |
| "logits/rejected": -3.138596773147583, | |
| "logps/chosen": -0.7956336140632629, | |
| "logps/rejected": -1.0347424745559692, | |
| "loss": 0.8565, | |
| "odds_ratio_loss": 0.6086810827255249, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.07956336438655853, | |
| "rewards/margins": 0.02391088381409645, | |
| "rewards/rejected": -0.10347424447536469, | |
| "sft_loss": 0.7956336140632629, | |
| "step": 960 | |
| }, | |
| { | |
| "epoch": 1.642675698560542, | |
| "grad_norm": 2.1293139457702637, | |
| "learning_rate": 2.1244588404994648e-06, | |
| "logits/chosen": -3.0621867179870605, | |
| "logits/rejected": -3.0614800453186035, | |
| "logps/chosen": -0.7734932899475098, | |
| "logps/rejected": -0.9287079572677612, | |
| "loss": 0.8408, | |
| "odds_ratio_loss": 0.6727171540260315, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.07734932750463486, | |
| "rewards/margins": 0.015521461144089699, | |
| "rewards/rejected": -0.0928707867860794, | |
| "sft_loss": 0.7734932899475098, | |
| "step": 970 | |
| }, | |
| { | |
| "epoch": 1.65961049957663, | |
| "grad_norm": 5.466368675231934, | |
| "learning_rate": 2.08064853874936e-06, | |
| "logits/chosen": -3.0892724990844727, | |
| "logits/rejected": -3.1403141021728516, | |
| "logps/chosen": -0.8142075538635254, | |
| "logps/rejected": -0.9884878396987915, | |
| "loss": 0.8786, | |
| "odds_ratio_loss": 0.6434410810470581, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.08142076432704926, | |
| "rewards/margins": 0.017428018152713776, | |
| "rewards/rejected": -0.09884877502918243, | |
| "sft_loss": 0.8142075538635254, | |
| "step": 980 | |
| }, | |
| { | |
| "epoch": 1.676545300592718, | |
| "grad_norm": 12.251886367797852, | |
| "learning_rate": 2.0369703571452387e-06, | |
| "logits/chosen": -3.055989980697632, | |
| "logits/rejected": -3.0450901985168457, | |
| "logps/chosen": -0.716150164604187, | |
| "logps/rejected": -0.9780189394950867, | |
| "loss": 0.7743, | |
| "odds_ratio_loss": 0.5815138816833496, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.07161502540111542, | |
| "rewards/margins": 0.026186879724264145, | |
| "rewards/rejected": -0.09780190140008926, | |
| "sft_loss": 0.716150164604187, | |
| "step": 990 | |
| }, | |
| { | |
| "epoch": 1.6934801016088061, | |
| "grad_norm": 3.6831836700439453, | |
| "learning_rate": 1.993438056859441e-06, | |
| "logits/chosen": -3.1155529022216797, | |
| "logits/rejected": -3.1060287952423096, | |
| "logps/chosen": -0.7484423518180847, | |
| "logps/rejected": -0.929480254650116, | |
| "loss": 0.8098, | |
| "odds_ratio_loss": 0.6138982772827148, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.07484424859285355, | |
| "rewards/margins": 0.018103793263435364, | |
| "rewards/rejected": -0.09294802695512772, | |
| "sft_loss": 0.7484423518180847, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 1.6934801016088061, | |
| "eval_logits/chosen": -3.0966453552246094, | |
| "eval_logits/rejected": -3.117032051086426, | |
| "eval_logps/chosen": -0.7911127805709839, | |
| "eval_logps/rejected": -0.9985377192497253, | |
| "eval_loss": 0.8548597693443298, | |
| "eval_odds_ratio_loss": 0.6374707221984863, | |
| "eval_rewards/accuracies": 0.5676190257072449, | |
| "eval_rewards/chosen": -0.07911127805709839, | |
| "eval_rewards/margins": 0.020742492750287056, | |
| "eval_rewards/rejected": -0.0998537689447403, | |
| "eval_runtime": 195.0995, | |
| "eval_samples_per_second": 5.382, | |
| "eval_sft_loss": 0.7911127805709839, | |
| "eval_steps_per_second": 2.691, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 1.710414902624894, | |
| "grad_norm": 2.5681636333465576, | |
| "learning_rate": 1.9500653531031917e-06, | |
| "logits/chosen": -3.116891384124756, | |
| "logits/rejected": -3.1181325912475586, | |
| "logps/chosen": -0.8085691332817078, | |
| "logps/rejected": -1.0325000286102295, | |
| "loss": 0.8768, | |
| "odds_ratio_loss": 0.6825646162033081, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.08085691183805466, | |
| "rewards/margins": 0.022393101826310158, | |
| "rewards/rejected": -0.10325001180171967, | |
| "sft_loss": 0.8085691332817078, | |
| "step": 1010 | |
| }, | |
| { | |
| "epoch": 1.7273497036409822, | |
| "grad_norm": 2.5795137882232666, | |
| "learning_rate": 1.9068659108055117e-06, | |
| "logits/chosen": -3.1321749687194824, | |
| "logits/rejected": -3.155836582183838, | |
| "logps/chosen": -0.7755477428436279, | |
| "logps/rejected": -0.9410096406936646, | |
| "loss": 0.8406, | |
| "odds_ratio_loss": 0.6507992744445801, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.07755477726459503, | |
| "rewards/margins": 0.016546186059713364, | |
| "rewards/rejected": -0.0941009670495987, | |
| "sft_loss": 0.7755477428436279, | |
| "step": 1020 | |
| }, | |
| { | |
| "epoch": 1.7442845046570703, | |
| "grad_norm": 2.360201835632324, | |
| "learning_rate": 1.863853340307962e-06, | |
| "logits/chosen": -3.078691005706787, | |
| "logits/rejected": -3.1044058799743652, | |
| "logps/chosen": -0.683570146560669, | |
| "logps/rejected": -0.9945551753044128, | |
| "loss": 0.7414, | |
| "odds_ratio_loss": 0.5782071352005005, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.06835701316595078, | |
| "rewards/margins": 0.03109849989414215, | |
| "rewards/rejected": -0.09945552051067352, | |
| "sft_loss": 0.683570146560669, | |
| "step": 1030 | |
| }, | |
| { | |
| "epoch": 1.7612193056731584, | |
| "grad_norm": 2.180799961090088, | |
| "learning_rate": 1.8210411930766019e-06, | |
| "logits/chosen": -3.0688931941986084, | |
| "logits/rejected": -3.096926212310791, | |
| "logps/chosen": -0.7589991688728333, | |
| "logps/rejected": -0.9845136404037476, | |
| "loss": 0.8196, | |
| "odds_ratio_loss": 0.6064754128456116, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.07589991390705109, | |
| "rewards/margins": 0.02255145087838173, | |
| "rewards/rejected": -0.09845136106014252, | |
| "sft_loss": 0.7589991688728333, | |
| "step": 1040 | |
| }, | |
| { | |
| "epoch": 1.7781541066892466, | |
| "grad_norm": 2.4373152256011963, | |
| "learning_rate": 1.7784429574324803e-06, | |
| "logits/chosen": -3.079857349395752, | |
| "logits/rejected": -3.1024343967437744, | |
| "logps/chosen": -0.738146960735321, | |
| "logps/rejected": -0.9651134610176086, | |
| "loss": 0.7991, | |
| "odds_ratio_loss": 0.6093234419822693, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.07381470501422882, | |
| "rewards/margins": 0.022696642205119133, | |
| "rewards/rejected": -0.0965113490819931, | |
| "sft_loss": 0.738146960735321, | |
| "step": 1050 | |
| }, | |
| { | |
| "epoch": 1.7950889077053345, | |
| "grad_norm": 1.4168922901153564, | |
| "learning_rate": 1.7360720543020327e-06, | |
| "logits/chosen": -3.158327102661133, | |
| "logits/rejected": -3.1482691764831543, | |
| "logps/chosen": -0.7429002523422241, | |
| "logps/rejected": -0.9451411366462708, | |
| "loss": 0.8021, | |
| "odds_ratio_loss": 0.5917203426361084, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.07429002225399017, | |
| "rewards/margins": 0.020224085077643394, | |
| "rewards/rejected": -0.09451410919427872, | |
| "sft_loss": 0.7429002523422241, | |
| "step": 1060 | |
| }, | |
| { | |
| "epoch": 1.8120237087214224, | |
| "grad_norm": 7.384251594543457, | |
| "learning_rate": 1.6939418329887042e-06, | |
| "logits/chosen": -3.1223366260528564, | |
| "logits/rejected": -3.157349109649658, | |
| "logps/chosen": -0.7720141410827637, | |
| "logps/rejected": -1.004219651222229, | |
| "loss": 0.8344, | |
| "odds_ratio_loss": 0.623358428478241, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.07720141112804413, | |
| "rewards/margins": 0.023220548406243324, | |
| "rewards/rejected": -0.1004219651222229, | |
| "sft_loss": 0.7720141410827637, | |
| "step": 1070 | |
| }, | |
| { | |
| "epoch": 1.8289585097375105, | |
| "grad_norm": 1.976076602935791, | |
| "learning_rate": 1.6520655669671467e-06, | |
| "logits/chosen": -3.151508331298828, | |
| "logits/rejected": -3.174203395843506, | |
| "logps/chosen": -0.7854975461959839, | |
| "logps/rejected": -0.9663504362106323, | |
| "loss": 0.8499, | |
| "odds_ratio_loss": 0.6441539525985718, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.07854975759983063, | |
| "rewards/margins": 0.01808529533445835, | |
| "rewards/rejected": -0.09663505107164383, | |
| "sft_loss": 0.7854975461959839, | |
| "step": 1080 | |
| }, | |
| { | |
| "epoch": 1.8458933107535986, | |
| "grad_norm": 2.3342175483703613, | |
| "learning_rate": 1.610456449701294e-06, | |
| "logits/chosen": -3.110039234161377, | |
| "logits/rejected": -3.1346383094787598, | |
| "logps/chosen": -0.7955976724624634, | |
| "logps/rejected": -1.0117195844650269, | |
| "loss": 0.8634, | |
| "odds_ratio_loss": 0.6780521273612976, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.07955978065729141, | |
| "rewards/margins": 0.02161218598484993, | |
| "rewards/rejected": -0.10117195546627045, | |
| "sft_loss": 0.7955976724624634, | |
| "step": 1090 | |
| }, | |
| { | |
| "epoch": 1.8628281117696868, | |
| "grad_norm": 2.040372133255005, | |
| "learning_rate": 1.5691275904876545e-06, | |
| "logits/chosen": -3.11810302734375, | |
| "logits/rejected": -3.0907464027404785, | |
| "logps/chosen": -0.8171418905258179, | |
| "logps/rejected": -1.049744963645935, | |
| "loss": 0.8794, | |
| "odds_ratio_loss": 0.6221813559532166, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.0817141979932785, | |
| "rewards/margins": 0.02326030097901821, | |
| "rewards/rejected": -0.10497449338436127, | |
| "sft_loss": 0.8171418905258179, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 1.879762912785775, | |
| "grad_norm": 1.1363812685012817, | |
| "learning_rate": 1.5280920103251235e-06, | |
| "logits/chosen": -3.121904134750366, | |
| "logits/rejected": -3.1400184631347656, | |
| "logps/chosen": -0.7571579217910767, | |
| "logps/rejected": -0.9578372240066528, | |
| "loss": 0.8213, | |
| "odds_ratio_loss": 0.6416117548942566, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.07571578770875931, | |
| "rewards/margins": 0.020067930221557617, | |
| "rewards/rejected": -0.09578372538089752, | |
| "sft_loss": 0.7571579217910767, | |
| "step": 1110 | |
| }, | |
| { | |
| "epoch": 1.8966977138018628, | |
| "grad_norm": 3.5236151218414307, | |
| "learning_rate": 1.4873626378126015e-06, | |
| "logits/chosen": -3.109051465988159, | |
| "logits/rejected": -3.123108386993408, | |
| "logps/chosen": -0.7997997403144836, | |
| "logps/rejected": -1.0338528156280518, | |
| "loss": 0.8642, | |
| "odds_ratio_loss": 0.6441462635993958, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.07997997850179672, | |
| "rewards/margins": 0.02340531349182129, | |
| "rewards/rejected": -0.10338529199361801, | |
| "sft_loss": 0.7997997403144836, | |
| "step": 1120 | |
| }, | |
| { | |
| "epoch": 1.913632514817951, | |
| "grad_norm": 4.680517673492432, | |
| "learning_rate": 1.446952305075738e-06, | |
| "logits/chosen": -3.1131389141082764, | |
| "logits/rejected": -3.151978015899658, | |
| "logps/chosen": -0.7748031616210938, | |
| "logps/rejected": -0.8985496759414673, | |
| "loss": 0.8402, | |
| "odds_ratio_loss": 0.6542429327964783, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.07748032361268997, | |
| "rewards/margins": 0.012374645099043846, | |
| "rewards/rejected": -0.08985497057437897, | |
| "sft_loss": 0.7748031616210938, | |
| "step": 1130 | |
| }, | |
| { | |
| "epoch": 1.9305673158340388, | |
| "grad_norm": 1.990546703338623, | |
| "learning_rate": 1.406873743724065e-06, | |
| "logits/chosen": -3.134593963623047, | |
| "logits/rejected": -3.088059186935425, | |
| "logps/chosen": -0.8194819688796997, | |
| "logps/rejected": -1.0895836353302002, | |
| "loss": 0.8842, | |
| "odds_ratio_loss": 0.6472653746604919, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.08194819837808609, | |
| "rewards/margins": 0.02701016701757908, | |
| "rewards/rejected": -0.10895836353302002, | |
| "sft_loss": 0.8194819688796997, | |
| "step": 1140 | |
| }, | |
| { | |
| "epoch": 1.947502116850127, | |
| "grad_norm": 2.0966198444366455, | |
| "learning_rate": 1.3671395808397898e-06, | |
| "logits/chosen": -3.1003191471099854, | |
| "logits/rejected": -3.159738779067993, | |
| "logps/chosen": -0.7502952218055725, | |
| "logps/rejected": -0.8763904571533203, | |
| "loss": 0.8142, | |
| "odds_ratio_loss": 0.6395031213760376, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.07502951472997665, | |
| "rewards/margins": 0.012609531171619892, | |
| "rewards/rejected": -0.08763904869556427, | |
| "sft_loss": 0.7502952218055725, | |
| "step": 1150 | |
| }, | |
| { | |
| "epoch": 1.964436917866215, | |
| "grad_norm": 6.19160795211792, | |
| "learning_rate": 1.3277623349995418e-06, | |
| "logits/chosen": -3.1115336418151855, | |
| "logits/rejected": -3.1329808235168457, | |
| "logps/chosen": -0.7726483941078186, | |
| "logps/rejected": -0.9365378618240356, | |
| "loss": 0.8381, | |
| "odds_ratio_loss": 0.6545372605323792, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.07726484537124634, | |
| "rewards/margins": 0.016388945281505585, | |
| "rewards/rejected": -0.09365378320217133, | |
| "sft_loss": 0.7726483941078186, | |
| "step": 1160 | |
| }, | |
| { | |
| "epoch": 1.9813717188823032, | |
| "grad_norm": 2.1759722232818604, | |
| "learning_rate": 1.2887544123302781e-06, | |
| "logits/chosen": -3.1343424320220947, | |
| "logits/rejected": -3.145904064178467, | |
| "logps/chosen": -0.795665442943573, | |
| "logps/rejected": -0.9278801679611206, | |
| "loss": 0.863, | |
| "odds_ratio_loss": 0.6731692552566528, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.07956655323505402, | |
| "rewards/margins": 0.01322146225720644, | |
| "rewards/rejected": -0.09278801828622818, | |
| "sft_loss": 0.795665442943573, | |
| "step": 1170 | |
| }, | |
| { | |
| "epoch": 1.9983065198983911, | |
| "grad_norm": 2.340391159057617, | |
| "learning_rate": 1.2501281026006393e-06, | |
| "logits/chosen": -3.113882064819336, | |
| "logits/rejected": -3.1493611335754395, | |
| "logps/chosen": -0.7624078989028931, | |
| "logps/rejected": -0.9082571268081665, | |
| "loss": 0.8289, | |
| "odds_ratio_loss": 0.6652835607528687, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.07624078541994095, | |
| "rewards/margins": 0.014584928750991821, | |
| "rewards/rejected": -0.09082571417093277, | |
| "sft_loss": 0.7624078989028931, | |
| "step": 1180 | |
| }, | |
| { | |
| "epoch": 2.015241320914479, | |
| "grad_norm": 2.1316022872924805, | |
| "learning_rate": 1.2118955753489523e-06, | |
| "logits/chosen": -3.1328041553497314, | |
| "logits/rejected": -3.119529962539673, | |
| "logps/chosen": -0.7868901491165161, | |
| "logps/rejected": -0.9673662185668945, | |
| "loss": 0.8514, | |
| "odds_ratio_loss": 0.6448970437049866, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.07868902385234833, | |
| "rewards/margins": 0.018047606572508812, | |
| "rewards/rejected": -0.09673662483692169, | |
| "sft_loss": 0.7868901491165161, | |
| "step": 1190 | |
| }, | |
| { | |
| "epoch": 2.032176121930567, | |
| "grad_norm": 2.219909906387329, | |
| "learning_rate": 1.1740688760491189e-06, | |
| "logits/chosen": -3.1113944053649902, | |
| "logits/rejected": -3.1537253856658936, | |
| "logps/chosen": -0.7722674608230591, | |
| "logps/rejected": -1.0175268650054932, | |
| "loss": 0.8299, | |
| "odds_ratio_loss": 0.5762220621109009, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.07722674310207367, | |
| "rewards/margins": 0.02452593669295311, | |
| "rewards/rejected": -0.10175268352031708, | |
| "sft_loss": 0.7722674608230591, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 2.0491109229466553, | |
| "grad_norm": 6.625383377075195, | |
| "learning_rate": 1.1366599223155847e-06, | |
| "logits/chosen": -3.092228651046753, | |
| "logits/rejected": -3.137305498123169, | |
| "logps/chosen": -0.7458280324935913, | |
| "logps/rejected": -1.1138603687286377, | |
| "loss": 0.8056, | |
| "odds_ratio_loss": 0.5978988409042358, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.07458280026912689, | |
| "rewards/margins": 0.0368032269179821, | |
| "rewards/rejected": -0.1113860234618187, | |
| "sft_loss": 0.7458280324935913, | |
| "step": 1210 | |
| }, | |
| { | |
| "epoch": 2.0660457239627434, | |
| "grad_norm": 3.234821081161499, | |
| "learning_rate": 1.0996805001486067e-06, | |
| "logits/chosen": -3.109764575958252, | |
| "logits/rejected": -3.1344008445739746, | |
| "logps/chosen": -0.7534822225570679, | |
| "logps/rejected": -0.9295659065246582, | |
| "loss": 0.8161, | |
| "odds_ratio_loss": 0.625883162021637, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.07534822076559067, | |
| "rewards/margins": 0.017608370631933212, | |
| "rewards/rejected": -0.09295658767223358, | |
| "sft_loss": 0.7534822225570679, | |
| "step": 1220 | |
| }, | |
| { | |
| "epoch": 2.0829805249788316, | |
| "grad_norm": 3.5898754596710205, | |
| "learning_rate": 1.0631422602209608e-06, | |
| "logits/chosen": -3.1376945972442627, | |
| "logits/rejected": -3.1564555168151855, | |
| "logps/chosen": -0.7927883863449097, | |
| "logps/rejected": -0.9650642275810242, | |
| "loss": 0.8569, | |
| "odds_ratio_loss": 0.6410055160522461, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.0792788416147232, | |
| "rewards/margins": 0.017227591946721077, | |
| "rewards/rejected": -0.09650643169879913, | |
| "sft_loss": 0.7927883863449097, | |
| "step": 1230 | |
| }, | |
| { | |
| "epoch": 2.0999153259949197, | |
| "grad_norm": 1.6415690183639526, | |
| "learning_rate": 1.027056714207319e-06, | |
| "logits/chosen": -3.1497724056243896, | |
| "logits/rejected": -3.1502339839935303, | |
| "logps/chosen": -0.8299382925033569, | |
| "logps/rejected": -1.0621525049209595, | |
| "loss": 0.8939, | |
| "odds_ratio_loss": 0.6398864984512329, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08299383521080017, | |
| "rewards/margins": 0.023221401497721672, | |
| "rewards/rejected": -0.10621523857116699, | |
| "sft_loss": 0.8299382925033569, | |
| "step": 1240 | |
| }, | |
| { | |
| "epoch": 2.116850127011008, | |
| "grad_norm": 2.1568691730499268, | |
| "learning_rate": 9.914352311573838e-07, | |
| "logits/chosen": -3.1183249950408936, | |
| "logits/rejected": -3.130068302154541, | |
| "logps/chosen": -0.680639386177063, | |
| "logps/rejected": -0.9202540516853333, | |
| "loss": 0.7394, | |
| "odds_ratio_loss": 0.5878284573554993, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.06806393712759018, | |
| "rewards/margins": 0.02396145835518837, | |
| "rewards/rejected": -0.09202539175748825, | |
| "sft_loss": 0.680639386177063, | |
| "step": 1250 | |
| }, | |
| { | |
| "epoch": 2.1337849280270955, | |
| "grad_norm": 1.6225751638412476, | |
| "learning_rate": 9.562890339139877e-07, | |
| "logits/chosen": -3.0915961265563965, | |
| "logits/rejected": -3.1580090522766113, | |
| "logps/chosen": -0.7393635511398315, | |
| "logps/rejected": -0.9581485986709595, | |
| "loss": 0.8032, | |
| "odds_ratio_loss": 0.6382402181625366, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.07393636554479599, | |
| "rewards/margins": 0.021878493949770927, | |
| "rewards/rejected": -0.09581486135721207, | |
| "sft_loss": 0.7393635511398315, | |
| "step": 1260 | |
| }, | |
| { | |
| "epoch": 2.1507197290431836, | |
| "grad_norm": 1.8676691055297852, | |
| "learning_rate": 9.216291955772374e-07, | |
| "logits/chosen": -3.0996224880218506, | |
| "logits/rejected": -3.153738021850586, | |
| "logps/chosen": -0.7607396841049194, | |
| "logps/rejected": -0.9290043115615845, | |
| "loss": 0.8238, | |
| "odds_ratio_loss": 0.6303194761276245, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.07607396692037582, | |
| "rewards/margins": 0.016826456412672997, | |
| "rewards/rejected": -0.09290042519569397, | |
| "sft_loss": 0.7607396841049194, | |
| "step": 1270 | |
| }, | |
| { | |
| "epoch": 2.167654530059272, | |
| "grad_norm": 2.096705198287964, | |
| "learning_rate": 8.874666360158457e-07, | |
| "logits/chosen": -3.082366466522217, | |
| "logits/rejected": -3.0917139053344727, | |
| "logps/chosen": -0.7106717824935913, | |
| "logps/rejected": -0.9525697827339172, | |
| "loss": 0.774, | |
| "odds_ratio_loss": 0.6332431435585022, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.07106717675924301, | |
| "rewards/margins": 0.024189796298742294, | |
| "rewards/rejected": -0.0952569767832756, | |
| "sft_loss": 0.7106717824935913, | |
| "step": 1280 | |
| }, | |
| { | |
| "epoch": 2.18458933107536, | |
| "grad_norm": 2.435957431793213, | |
| "learning_rate": 8.538121184267315e-07, | |
| "logits/chosen": -3.1009063720703125, | |
| "logits/rejected": -3.121425151824951, | |
| "logps/chosen": -0.6681427955627441, | |
| "logps/rejected": -0.9359544515609741, | |
| "loss": 0.7277, | |
| "odds_ratio_loss": 0.595305323600769, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.06681428104639053, | |
| "rewards/margins": 0.026781165972352028, | |
| "rewards/rejected": -0.09359544515609741, | |
| "sft_loss": 0.6681427955627441, | |
| "step": 1290 | |
| }, | |
| { | |
| "epoch": 2.201524132091448, | |
| "grad_norm": 2.3414995670318604, | |
| "learning_rate": 8.206762459439907e-07, | |
| "logits/chosen": -3.1007769107818604, | |
| "logits/rejected": -3.142437696456909, | |
| "logps/chosen": -0.7944619059562683, | |
| "logps/rejected": -0.9959940910339355, | |
| "loss": 0.8604, | |
| "odds_ratio_loss": 0.659730076789856, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.07944619655609131, | |
| "rewards/margins": 0.020153220742940903, | |
| "rewards/rejected": -0.09959942102432251, | |
| "sft_loss": 0.7944619059562683, | |
| "step": 1300 | |
| }, | |
| { | |
| "epoch": 2.218458933107536, | |
| "grad_norm": 2.9054501056671143, | |
| "learning_rate": 7.880694582982898e-07, | |
| "logits/chosen": -3.165544271469116, | |
| "logits/rejected": -3.1803879737854004, | |
| "logps/chosen": -0.8287284970283508, | |
| "logps/rejected": -1.0228365659713745, | |
| "loss": 0.8926, | |
| "odds_ratio_loss": 0.6389774084091187, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.08287284523248672, | |
| "rewards/margins": 0.019410807639360428, | |
| "rewards/rejected": -0.10228364169597626, | |
| "sft_loss": 0.8287284970283508, | |
| "step": 1310 | |
| }, | |
| { | |
| "epoch": 2.235393734123624, | |
| "grad_norm": 3.0612242221832275, | |
| "learning_rate": 7.560020285277401e-07, | |
| "logits/chosen": -3.0883891582489014, | |
| "logits/rejected": -3.142484426498413, | |
| "logps/chosen": -0.7654698491096497, | |
| "logps/rejected": -0.8691753149032593, | |
| "loss": 0.8363, | |
| "odds_ratio_loss": 0.708137035369873, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.07654698193073273, | |
| "rewards/margins": 0.010370554402470589, | |
| "rewards/rejected": -0.08691753447055817, | |
| "sft_loss": 0.7654698491096497, | |
| "step": 1320 | |
| }, | |
| { | |
| "epoch": 2.252328535139712, | |
| "grad_norm": 3.074390172958374, | |
| "learning_rate": 7.244840597412956e-07, | |
| "logits/chosen": -3.0840373039245605, | |
| "logits/rejected": -3.113865613937378, | |
| "logps/chosen": -0.7826686501502991, | |
| "logps/rejected": -0.8837703466415405, | |
| "loss": 0.8534, | |
| "odds_ratio_loss": 0.7077327370643616, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.07826686650514603, | |
| "rewards/margins": 0.010110177099704742, | |
| "rewards/rejected": -0.08837703615427017, | |
| "sft_loss": 0.7826686501502991, | |
| "step": 1330 | |
| }, | |
| { | |
| "epoch": 2.2692633361558, | |
| "grad_norm": 6.461457252502441, | |
| "learning_rate": 6.935254819356796e-07, | |
| "logits/chosen": -3.1297388076782227, | |
| "logits/rejected": -3.1470894813537598, | |
| "logps/chosen": -0.7817297577857971, | |
| "logps/rejected": -0.9597527384757996, | |
| "loss": 0.8464, | |
| "odds_ratio_loss": 0.6471723318099976, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.07817298173904419, | |
| "rewards/margins": 0.017802301794290543, | |
| "rewards/rejected": -0.09597527980804443, | |
| "sft_loss": 0.7817297577857971, | |
| "step": 1340 | |
| }, | |
| { | |
| "epoch": 2.2861981371718882, | |
| "grad_norm": 1.7192656993865967, | |
| "learning_rate": 6.631360488668662e-07, | |
| "logits/chosen": -3.138521194458008, | |
| "logits/rejected": -3.150005578994751, | |
| "logps/chosen": -0.7190467715263367, | |
| "logps/rejected": -0.9147864580154419, | |
| "loss": 0.7824, | |
| "odds_ratio_loss": 0.6330953240394592, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.07190467417240143, | |
| "rewards/margins": 0.019573967903852463, | |
| "rewards/rejected": -0.09147863835096359, | |
| "sft_loss": 0.7190467715263367, | |
| "step": 1350 | |
| }, | |
| { | |
| "epoch": 2.3031329381879764, | |
| "grad_norm": 2.849228620529175, | |
| "learning_rate": 6.333253349770672e-07, | |
| "logits/chosen": -3.1443207263946533, | |
| "logits/rejected": -3.1502537727355957, | |
| "logps/chosen": -0.8002703785896301, | |
| "logps/rejected": -0.9851358532905579, | |
| "loss": 0.8656, | |
| "odds_ratio_loss": 0.6537164449691772, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.0800270363688469, | |
| "rewards/margins": 0.018486548215150833, | |
| "rewards/rejected": -0.09851358830928802, | |
| "sft_loss": 0.8002703785896301, | |
| "step": 1360 | |
| }, | |
| { | |
| "epoch": 2.3200677392040645, | |
| "grad_norm": 1.3985761404037476, | |
| "learning_rate": 6.041027323782364e-07, | |
| "logits/chosen": -3.136557102203369, | |
| "logits/rejected": -3.1552491188049316, | |
| "logps/chosen": -0.7381452918052673, | |
| "logps/rejected": -0.9497495889663696, | |
| "loss": 0.7993, | |
| "odds_ratio_loss": 0.611694872379303, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.07381454110145569, | |
| "rewards/margins": 0.021160420030355453, | |
| "rewards/rejected": -0.09497495740652084, | |
| "sft_loss": 0.7381452918052673, | |
| "step": 1370 | |
| }, | |
| { | |
| "epoch": 2.337002540220152, | |
| "grad_norm": 2.8737893104553223, | |
| "learning_rate": 5.754774478929969e-07, | |
| "logits/chosen": -3.1531028747558594, | |
| "logits/rejected": -3.1830787658691406, | |
| "logps/chosen": -0.7512658834457397, | |
| "logps/rejected": -0.9563377499580383, | |
| "loss": 0.81, | |
| "odds_ratio_loss": 0.5872438549995422, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.07512658089399338, | |
| "rewards/margins": 0.020507195964455605, | |
| "rewards/rejected": -0.09563378244638443, | |
| "sft_loss": 0.7512658834457397, | |
| "step": 1380 | |
| }, | |
| { | |
| "epoch": 2.3539373412362403, | |
| "grad_norm": 3.9828193187713623, | |
| "learning_rate": 5.474585001539634e-07, | |
| "logits/chosen": -3.159769296646118, | |
| "logits/rejected": -3.1827354431152344, | |
| "logps/chosen": -0.716742753982544, | |
| "logps/rejected": -0.9266722798347473, | |
| "loss": 0.7742, | |
| "odds_ratio_loss": 0.5742050409317017, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.07167427241802216, | |
| "rewards/margins": 0.020992957055568695, | |
| "rewards/rejected": -0.09266723692417145, | |
| "sft_loss": 0.716742753982544, | |
| "step": 1390 | |
| }, | |
| { | |
| "epoch": 2.3708721422523285, | |
| "grad_norm": 1.6741605997085571, | |
| "learning_rate": 5.200547167623424e-07, | |
| "logits/chosen": -3.172938823699951, | |
| "logits/rejected": -3.1454200744628906, | |
| "logps/chosen": -0.7799001932144165, | |
| "logps/rejected": -1.040466070175171, | |
| "loss": 0.8399, | |
| "odds_ratio_loss": 0.5997284650802612, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.07799001783132553, | |
| "rewards/margins": 0.026056593284010887, | |
| "rewards/rejected": -0.10404660552740097, | |
| "sft_loss": 0.7799001932144165, | |
| "step": 1400 | |
| }, | |
| { | |
| "epoch": 2.3878069432684166, | |
| "grad_norm": 1.3470762968063354, | |
| "learning_rate": 4.932747315067271e-07, | |
| "logits/chosen": -3.15238356590271, | |
| "logits/rejected": -3.1575927734375, | |
| "logps/chosen": -0.7618133425712585, | |
| "logps/rejected": -1.033060073852539, | |
| "loss": 0.8193, | |
| "odds_ratio_loss": 0.574641764163971, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.0761813372373581, | |
| "rewards/margins": 0.027124667540192604, | |
| "rewards/rejected": -0.10330601036548615, | |
| "sft_loss": 0.7618133425712585, | |
| "step": 1410 | |
| }, | |
| { | |
| "epoch": 2.4047417442845047, | |
| "grad_norm": 4.0449371337890625, | |
| "learning_rate": 4.6712698164294553e-07, | |
| "logits/chosen": -3.1169888973236084, | |
| "logits/rejected": -3.131412982940674, | |
| "logps/chosen": -0.7451744079589844, | |
| "logps/rejected": -0.9645574688911438, | |
| "loss": 0.8029, | |
| "odds_ratio_loss": 0.5777136087417603, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.07451744377613068, | |
| "rewards/margins": 0.021938303485512733, | |
| "rewards/rejected": -0.09645574539899826, | |
| "sft_loss": 0.7451744079589844, | |
| "step": 1420 | |
| }, | |
| { | |
| "epoch": 2.421676545300593, | |
| "grad_norm": 1.97295343875885, | |
| "learning_rate": 4.41619705235842e-07, | |
| "logits/chosen": -3.1363072395324707, | |
| "logits/rejected": -3.1507456302642822, | |
| "logps/chosen": -0.7545329332351685, | |
| "logps/rejected": -1.082715392112732, | |
| "loss": 0.8124, | |
| "odds_ratio_loss": 0.5782482624053955, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.07545328885316849, | |
| "rewards/margins": 0.0328182615339756, | |
| "rewards/rejected": -0.10827155411243439, | |
| "sft_loss": 0.7545329332351685, | |
| "step": 1430 | |
| }, | |
| { | |
| "epoch": 2.438611346316681, | |
| "grad_norm": 2.880293846130371, | |
| "learning_rate": 4.167609385637961e-07, | |
| "logits/chosen": -3.182002544403076, | |
| "logits/rejected": -3.165118455886841, | |
| "logps/chosen": -0.7962235808372498, | |
| "logps/rejected": -0.975197970867157, | |
| "loss": 0.8596, | |
| "odds_ratio_loss": 0.634021520614624, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.07962235808372498, | |
| "rewards/margins": 0.017897438257932663, | |
| "rewards/rejected": -0.09751980006694794, | |
| "sft_loss": 0.7962235808372498, | |
| "step": 1440 | |
| }, | |
| { | |
| "epoch": 2.4555461473327687, | |
| "grad_norm": 2.545621871948242, | |
| "learning_rate": 3.9255851358683567e-07, | |
| "logits/chosen": -3.1265337467193604, | |
| "logits/rejected": -3.140825033187866, | |
| "logps/chosen": -0.7502083778381348, | |
| "logps/rejected": -0.9492311477661133, | |
| "loss": 0.817, | |
| "odds_ratio_loss": 0.6676734685897827, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.07502084225416183, | |
| "rewards/margins": 0.019902262836694717, | |
| "rewards/rejected": -0.09492311626672745, | |
| "sft_loss": 0.7502083778381348, | |
| "step": 1450 | |
| }, | |
| { | |
| "epoch": 2.472480948348857, | |
| "grad_norm": 2.91165828704834, | |
| "learning_rate": 3.690200554791082e-07, | |
| "logits/chosen": -3.093761444091797, | |
| "logits/rejected": -3.0926525592803955, | |
| "logps/chosen": -0.7423545122146606, | |
| "logps/rejected": -0.9597676992416382, | |
| "loss": 0.8001, | |
| "odds_ratio_loss": 0.5777243375778198, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.0742354542016983, | |
| "rewards/margins": 0.02174132689833641, | |
| "rewards/rejected": -0.09597676992416382, | |
| "sft_loss": 0.7423545122146606, | |
| "step": 1460 | |
| }, | |
| { | |
| "epoch": 2.489415749364945, | |
| "grad_norm": 5.057043075561523, | |
| "learning_rate": 3.461529802265079e-07, | |
| "logits/chosen": -3.153536796569824, | |
| "logits/rejected": -3.16294264793396, | |
| "logps/chosen": -0.7584110498428345, | |
| "logps/rejected": -0.9484812617301941, | |
| "loss": 0.8201, | |
| "odds_ratio_loss": 0.6163991093635559, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.07584110647439957, | |
| "rewards/margins": 0.01900702901184559, | |
| "rewards/rejected": -0.0948481336236, | |
| "sft_loss": 0.7584110498428345, | |
| "step": 1470 | |
| }, | |
| { | |
| "epoch": 2.506350550381033, | |
| "grad_norm": 2.0298070907592773, | |
| "learning_rate": 3.2396449229020883e-07, | |
| "logits/chosen": -3.18571400642395, | |
| "logits/rejected": -3.1676132678985596, | |
| "logps/chosen": -0.7937291264533997, | |
| "logps/rejected": -0.935733437538147, | |
| "loss": 0.8599, | |
| "odds_ratio_loss": 0.6614922881126404, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.07937291264533997, | |
| "rewards/margins": 0.014200428500771523, | |
| "rewards/rejected": -0.09357334673404694, | |
| "sft_loss": 0.7937291264533997, | |
| "step": 1480 | |
| }, | |
| { | |
| "epoch": 2.523285351397121, | |
| "grad_norm": 4.000706195831299, | |
| "learning_rate": 3.024615823368371e-07, | |
| "logits/chosen": -3.1206421852111816, | |
| "logits/rejected": -3.152639150619507, | |
| "logps/chosen": -0.7629178166389465, | |
| "logps/rejected": -0.9828959703445435, | |
| "loss": 0.8252, | |
| "odds_ratio_loss": 0.622687041759491, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.07629178464412689, | |
| "rewards/margins": 0.021997807547450066, | |
| "rewards/rejected": -0.09828958660364151, | |
| "sft_loss": 0.7629178166389465, | |
| "step": 1490 | |
| }, | |
| { | |
| "epoch": 2.5402201524132093, | |
| "grad_norm": 2.449856758117676, | |
| "learning_rate": 2.8165102503600716e-07, | |
| "logits/chosen": -3.095520496368408, | |
| "logits/rejected": -3.1078646183013916, | |
| "logps/chosen": -0.7517341375350952, | |
| "logps/rejected": -0.9984272718429565, | |
| "loss": 0.8135, | |
| "odds_ratio_loss": 0.6172657012939453, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.07517342269420624, | |
| "rewards/margins": 0.024669310078024864, | |
| "rewards/rejected": -0.09984272718429565, | |
| "sft_loss": 0.7517341375350952, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 2.5402201524132093, | |
| "eval_logits/chosen": -3.130527973175049, | |
| "eval_logits/rejected": -3.1507296562194824, | |
| "eval_logps/chosen": -0.7867480516433716, | |
| "eval_logps/rejected": -0.9955620169639587, | |
| "eval_loss": 0.8505691885948181, | |
| "eval_odds_ratio_loss": 0.638211727142334, | |
| "eval_rewards/accuracies": 0.5723809599876404, | |
| "eval_rewards/chosen": -0.0786748081445694, | |
| "eval_rewards/margins": 0.020881392061710358, | |
| "eval_rewards/rejected": -0.09955620020627975, | |
| "eval_runtime": 194.4899, | |
| "eval_samples_per_second": 5.399, | |
| "eval_sft_loss": 0.7867480516433716, | |
| "eval_steps_per_second": 2.699, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 2.557154953429297, | |
| "grad_norm": 1.5426675081253052, | |
| "learning_rate": 2.615393769259039e-07, | |
| "logits/chosen": -3.1200222969055176, | |
| "logits/rejected": -3.151517152786255, | |
| "logps/chosen": -0.845578670501709, | |
| "logps/rejected": -0.9381099939346313, | |
| "loss": 0.9194, | |
| "odds_ratio_loss": 0.7380752563476562, | |
| "rewards/accuracies": 0.4625000059604645, | |
| "rewards/chosen": -0.08455787599086761, | |
| "rewards/margins": 0.00925312377512455, | |
| "rewards/rejected": -0.09381099790334702, | |
| "sft_loss": 0.845578670501709, | |
| "step": 1510 | |
| }, | |
| { | |
| "epoch": 2.574089754445385, | |
| "grad_norm": 4.487434387207031, | |
| "learning_rate": 2.421329743475917e-07, | |
| "logits/chosen": -3.1358139514923096, | |
| "logits/rejected": -3.157161235809326, | |
| "logps/chosen": -0.7332046627998352, | |
| "logps/rejected": -0.9182003140449524, | |
| "loss": 0.7981, | |
| "odds_ratio_loss": 0.6488706469535828, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.07332046329975128, | |
| "rewards/margins": 0.018499553203582764, | |
| "rewards/rejected": -0.09182002395391464, | |
| "sft_loss": 0.7332046627998352, | |
| "step": 1520 | |
| }, | |
| { | |
| "epoch": 2.5910245554614733, | |
| "grad_norm": 2.3630266189575195, | |
| "learning_rate": 2.234379314486973e-07, | |
| "logits/chosen": -3.1165411472320557, | |
| "logits/rejected": -3.1539194583892822, | |
| "logps/chosen": -0.8104802370071411, | |
| "logps/rejected": -0.9215306043624878, | |
| "loss": 0.8756, | |
| "odds_ratio_loss": 0.6507223844528198, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08104802668094635, | |
| "rewards/margins": 0.011105035431683064, | |
| "rewards/rejected": -0.09215305000543594, | |
| "sft_loss": 0.8104802370071411, | |
| "step": 1530 | |
| }, | |
| { | |
| "epoch": 2.6079593564775614, | |
| "grad_norm": 1.243202805519104, | |
| "learning_rate": 2.0546013825709783e-07, | |
| "logits/chosen": -3.1084282398223877, | |
| "logits/rejected": -3.129692792892456, | |
| "logps/chosen": -0.7565353512763977, | |
| "logps/rejected": -1.0974582433700562, | |
| "loss": 0.8149, | |
| "odds_ratio_loss": 0.5834510326385498, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.07565353065729141, | |
| "rewards/margins": 0.034092292189598083, | |
| "rewards/rejected": -0.1097458228468895, | |
| "sft_loss": 0.7565353512763977, | |
| "step": 1540 | |
| }, | |
| { | |
| "epoch": 2.6248941574936495, | |
| "grad_norm": 1.8778706789016724, | |
| "learning_rate": 1.88205258825217e-07, | |
| "logits/chosen": -3.0997517108917236, | |
| "logits/rejected": -3.0937392711639404, | |
| "logps/chosen": -0.672328770160675, | |
| "logps/rejected": -0.9460701942443848, | |
| "loss": 0.7295, | |
| "odds_ratio_loss": 0.5718902349472046, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.06723286956548691, | |
| "rewards/margins": 0.02737414464354515, | |
| "rewards/rejected": -0.09460701793432236, | |
| "sft_loss": 0.672328770160675, | |
| "step": 1550 | |
| }, | |
| { | |
| "epoch": 2.6418289585097376, | |
| "grad_norm": 3.3697926998138428, | |
| "learning_rate": 1.7167872944552245e-07, | |
| "logits/chosen": -3.0951905250549316, | |
| "logits/rejected": -3.143221378326416, | |
| "logps/chosen": -0.6983746290206909, | |
| "logps/rejected": -0.9046236276626587, | |
| "loss": 0.7572, | |
| "odds_ratio_loss": 0.5877906084060669, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.06983745843172073, | |
| "rewards/margins": 0.020624909549951553, | |
| "rewards/rejected": -0.09046236425638199, | |
| "sft_loss": 0.6983746290206909, | |
| "step": 1560 | |
| }, | |
| { | |
| "epoch": 2.6587637595258258, | |
| "grad_norm": 2.3217389583587646, | |
| "learning_rate": 1.5588575693777142e-07, | |
| "logits/chosen": -3.1411678791046143, | |
| "logits/rejected": -3.1680665016174316, | |
| "logps/chosen": -0.7495251297950745, | |
| "logps/rejected": -0.8962133526802063, | |
| "loss": 0.8135, | |
| "odds_ratio_loss": 0.6393758058547974, | |
| "rewards/accuracies": 0.5062500238418579, | |
| "rewards/chosen": -0.07495252043008804, | |
| "rewards/margins": 0.01466882973909378, | |
| "rewards/rejected": -0.08962134271860123, | |
| "sft_loss": 0.7495251297950745, | |
| "step": 1570 | |
| }, | |
| { | |
| "epoch": 2.675698560541914, | |
| "grad_norm": 2.695345401763916, | |
| "learning_rate": 1.4083131700856428e-07, | |
| "logits/chosen": -3.093736410140991, | |
| "logits/rejected": -3.154810905456543, | |
| "logps/chosen": -0.7956855893135071, | |
| "logps/rejected": -0.9356697201728821, | |
| "loss": 0.8605, | |
| "odds_ratio_loss": 0.6480044722557068, | |
| "rewards/accuracies": 0.5249999761581421, | |
| "rewards/chosen": -0.07956856489181519, | |
| "rewards/margins": 0.013998406007885933, | |
| "rewards/rejected": -0.09356696903705597, | |
| "sft_loss": 0.7956855893135071, | |
| "step": 1580 | |
| }, | |
| { | |
| "epoch": 2.6926333615580016, | |
| "grad_norm": 4.358800888061523, | |
| "learning_rate": 1.2652015268370315e-07, | |
| "logits/chosen": -3.150224208831787, | |
| "logits/rejected": -3.183022975921631, | |
| "logps/chosen": -0.7435088753700256, | |
| "logps/rejected": -0.9757458567619324, | |
| "loss": 0.8044, | |
| "odds_ratio_loss": 0.6088230013847351, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.07435088604688644, | |
| "rewards/margins": 0.02322370745241642, | |
| "rewards/rejected": -0.09757460653781891, | |
| "sft_loss": 0.7435088753700256, | |
| "step": 1590 | |
| }, | |
| { | |
| "epoch": 2.7095681625740897, | |
| "grad_norm": 1.9499818086624146, | |
| "learning_rate": 1.1295677281386502e-07, | |
| "logits/chosen": -3.1664371490478516, | |
| "logits/rejected": -3.1615631580352783, | |
| "logps/chosen": -0.8278031349182129, | |
| "logps/rejected": -1.062387228012085, | |
| "loss": 0.8914, | |
| "odds_ratio_loss": 0.6362180113792419, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.08278031647205353, | |
| "rewards/margins": 0.02345840446650982, | |
| "rewards/rejected": -0.1062387228012085, | |
| "sft_loss": 0.8278031349182129, | |
| "step": 1600 | |
| }, | |
| { | |
| "epoch": 2.726502963590178, | |
| "grad_norm": 1.277694821357727, | |
| "learning_rate": 1.0014545065404973e-07, | |
| "logits/chosen": -3.12839674949646, | |
| "logits/rejected": -3.1691813468933105, | |
| "logps/chosen": -0.8073375821113586, | |
| "logps/rejected": -1.0353208780288696, | |
| "loss": 0.8777, | |
| "odds_ratio_loss": 0.7032862305641174, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.0807337611913681, | |
| "rewards/margins": 0.022798333317041397, | |
| "rewards/rejected": -0.1035320907831192, | |
| "sft_loss": 0.8073375821113586, | |
| "step": 1610 | |
| }, | |
| { | |
| "epoch": 2.743437764606266, | |
| "grad_norm": 1.525190830230713, | |
| "learning_rate": 8.809022251725502e-08, | |
| "logits/chosen": -3.1842474937438965, | |
| "logits/rejected": -3.156172752380371, | |
| "logps/chosen": -0.7726050615310669, | |
| "logps/rejected": -1.0311490297317505, | |
| "loss": 0.8348, | |
| "odds_ratio_loss": 0.6221681833267212, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.07726050913333893, | |
| "rewards/margins": 0.02585439942777157, | |
| "rewards/rejected": -0.10311490297317505, | |
| "sft_loss": 0.7726050615310669, | |
| "step": 1620 | |
| }, | |
| { | |
| "epoch": 2.7603725656223537, | |
| "grad_norm": 2.015005111694336, | |
| "learning_rate": 7.679488650280509e-08, | |
| "logits/chosen": -3.192863702774048, | |
| "logits/rejected": -3.213048219680786, | |
| "logps/chosen": -0.7830113172531128, | |
| "logps/rejected": -0.9780160188674927, | |
| "loss": 0.8429, | |
| "odds_ratio_loss": 0.5990433096885681, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.07830111682415009, | |
| "rewards/margins": 0.019500473514199257, | |
| "rewards/rejected": -0.09780160337686539, | |
| "sft_loss": 0.7830113172531128, | |
| "step": 1630 | |
| }, | |
| { | |
| "epoch": 2.777307366638442, | |
| "grad_norm": 2.5389175415039062, | |
| "learning_rate": 6.626300129972563e-08, | |
| "logits/chosen": -3.122981071472168, | |
| "logits/rejected": -3.214785099029541, | |
| "logps/chosen": -0.7296528220176697, | |
| "logps/rejected": -0.955167293548584, | |
| "loss": 0.792, | |
| "odds_ratio_loss": 0.6230596303939819, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.07296527922153473, | |
| "rewards/margins": 0.02255145087838173, | |
| "rewards/rejected": -0.09551674872636795, | |
| "sft_loss": 0.7296528220176697, | |
| "step": 1640 | |
| }, | |
| { | |
| "epoch": 2.79424216765453, | |
| "grad_norm": 1.7503160238265991, | |
| "learning_rate": 5.649788506555065e-08, | |
| "logits/chosen": -3.115635871887207, | |
| "logits/rejected": -3.166555881500244, | |
| "logps/chosen": -0.7383006811141968, | |
| "logps/rejected": -0.9469397664070129, | |
| "loss": 0.797, | |
| "odds_ratio_loss": 0.5870878100395203, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.07383007556200027, | |
| "rewards/margins": 0.020863894373178482, | |
| "rewards/rejected": -0.09469397366046906, | |
| "sft_loss": 0.7383006811141968, | |
| "step": 1650 | |
| }, | |
| { | |
| "epoch": 2.811176968670618, | |
| "grad_norm": 5.490494728088379, | |
| "learning_rate": 4.7502614380908474e-08, | |
| "logits/chosen": -3.1543877124786377, | |
| "logits/rejected": -3.134704113006592, | |
| "logps/chosen": -0.7694907188415527, | |
| "logps/rejected": -0.9553998112678528, | |
| "loss": 0.8333, | |
| "odds_ratio_loss": 0.6379188895225525, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.07694907486438751, | |
| "rewards/margins": 0.018590910360217094, | |
| "rewards/rejected": -0.09553998708724976, | |
| "sft_loss": 0.7694907188415527, | |
| "step": 1660 | |
| }, | |
| { | |
| "epoch": 2.828111769686706, | |
| "grad_norm": 3.784432888031006, | |
| "learning_rate": 3.9280023280222066e-08, | |
| "logits/chosen": -3.1404221057891846, | |
| "logits/rejected": -3.1622607707977295, | |
| "logps/chosen": -0.757176399230957, | |
| "logps/rejected": -0.9332196116447449, | |
| "loss": 0.8222, | |
| "odds_ratio_loss": 0.6507169008255005, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.07571764290332794, | |
| "rewards/margins": 0.017604324966669083, | |
| "rewards/rejected": -0.09332196414470673, | |
| "sft_loss": 0.757176399230957, | |
| "step": 1670 | |
| }, | |
| { | |
| "epoch": 2.8450465707027943, | |
| "grad_norm": 2.088921546936035, | |
| "learning_rate": 3.1832702358818855e-08, | |
| "logits/chosen": -3.116088390350342, | |
| "logits/rejected": -3.1639227867126465, | |
| "logps/chosen": -0.8132543563842773, | |
| "logps/rejected": -1.068440556526184, | |
| "loss": 0.8739, | |
| "odds_ratio_loss": 0.6062491536140442, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.08132544159889221, | |
| "rewards/margins": 0.025518611073493958, | |
| "rewards/rejected": -0.10684405267238617, | |
| "sft_loss": 0.8132543563842773, | |
| "step": 1680 | |
| }, | |
| { | |
| "epoch": 2.8619813717188824, | |
| "grad_norm": 2.5282764434814453, | |
| "learning_rate": 2.5162997956746647e-08, | |
| "logits/chosen": -3.1500442028045654, | |
| "logits/rejected": -3.171433210372925, | |
| "logps/chosen": -0.7479134202003479, | |
| "logps/rejected": -0.9974180459976196, | |
| "loss": 0.8076, | |
| "odds_ratio_loss": 0.596734344959259, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.0747913345694542, | |
| "rewards/margins": 0.02495047077536583, | |
| "rewards/rejected": -0.09974180907011032, | |
| "sft_loss": 0.7479134202003479, | |
| "step": 1690 | |
| }, | |
| { | |
| "epoch": 2.8789161727349706, | |
| "grad_norm": 3.469899892807007, | |
| "learning_rate": 1.9273011419536914e-08, | |
| "logits/chosen": -3.140916585922241, | |
| "logits/rejected": -3.16453218460083, | |
| "logps/chosen": -0.7706011533737183, | |
| "logps/rejected": -0.9274336695671082, | |
| "loss": 0.8378, | |
| "odds_ratio_loss": 0.6717150211334229, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.07706011831760406, | |
| "rewards/margins": 0.0156832467764616, | |
| "rewards/rejected": -0.09274337440729141, | |
| "sft_loss": 0.7706011533737183, | |
| "step": 1700 | |
| }, | |
| { | |
| "epoch": 2.8958509737510583, | |
| "grad_norm": 1.3444654941558838, | |
| "learning_rate": 1.4164598436159083e-08, | |
| "logits/chosen": -3.1658730506896973, | |
| "logits/rejected": -3.1828174591064453, | |
| "logps/chosen": -0.7568970322608948, | |
| "logps/rejected": -0.9151817560195923, | |
| "loss": 0.82, | |
| "odds_ratio_loss": 0.630837082862854, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.07568971067667007, | |
| "rewards/margins": 0.01582847535610199, | |
| "rewards/rejected": -0.09151817858219147, | |
| "sft_loss": 0.7568970322608948, | |
| "step": 1710 | |
| }, | |
| { | |
| "epoch": 2.9127857747671464, | |
| "grad_norm": 2.7227256298065186, | |
| "learning_rate": 9.839368454371556e-09, | |
| "logits/chosen": -3.1114795207977295, | |
| "logits/rejected": -3.11403751373291, | |
| "logps/chosen": -0.7409245371818542, | |
| "logps/rejected": -1.0069682598114014, | |
| "loss": 0.8046, | |
| "odds_ratio_loss": 0.6370644569396973, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.07409246265888214, | |
| "rewards/margins": 0.026604369282722473, | |
| "rewards/rejected": -0.10069682449102402, | |
| "sft_loss": 0.7409245371818542, | |
| "step": 1720 | |
| }, | |
| { | |
| "epoch": 2.9297205757832345, | |
| "grad_norm": 1.9127988815307617, | |
| "learning_rate": 6.298684173650649e-09, | |
| "logits/chosen": -3.0886847972869873, | |
| "logits/rejected": -3.123133897781372, | |
| "logps/chosen": -0.7336040735244751, | |
| "logps/rejected": -1.0557795763015747, | |
| "loss": 0.7977, | |
| "odds_ratio_loss": 0.640912652015686, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.07336040586233139, | |
| "rewards/margins": 0.03221754729747772, | |
| "rewards/rejected": -0.10557796061038971, | |
| "sft_loss": 0.7336040735244751, | |
| "step": 1730 | |
| }, | |
| { | |
| "epoch": 2.9466553767993227, | |
| "grad_norm": 4.324622631072998, | |
| "learning_rate": 3.543661115860686e-09, | |
| "logits/chosen": -3.103651523590088, | |
| "logits/rejected": -3.136838674545288, | |
| "logps/chosen": -0.790323793888092, | |
| "logps/rejected": -1.0052763223648071, | |
| "loss": 0.8552, | |
| "odds_ratio_loss": 0.6486952900886536, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.07903237640857697, | |
| "rewards/margins": 0.021495262160897255, | |
| "rewards/rejected": -0.10052764415740967, | |
| "sft_loss": 0.790323793888092, | |
| "step": 1740 | |
| }, | |
| { | |
| "epoch": 2.963590177815411, | |
| "grad_norm": 2.1066298484802246, | |
| "learning_rate": 1.575167273800693e-09, | |
| "logits/chosen": -3.1322402954101562, | |
| "logits/rejected": -3.141758680343628, | |
| "logps/chosen": -0.7295509576797485, | |
| "logps/rejected": -0.8775620460510254, | |
| "loss": 0.7914, | |
| "odds_ratio_loss": 0.6189672350883484, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.07295509427785873, | |
| "rewards/margins": 0.01480111014097929, | |
| "rewards/rejected": -0.0877562090754509, | |
| "sft_loss": 0.7295509576797485, | |
| "step": 1750 | |
| }, | |
| { | |
| "epoch": 2.9805249788314985, | |
| "grad_norm": 2.692870855331421, | |
| "learning_rate": 3.9382283773564676e-10, | |
| "logits/chosen": -3.153014659881592, | |
| "logits/rejected": -3.1698267459869385, | |
| "logps/chosen": -0.8286467790603638, | |
| "logps/rejected": -1.0176646709442139, | |
| "loss": 0.8971, | |
| "odds_ratio_loss": 0.6844185590744019, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.0828646719455719, | |
| "rewards/margins": 0.018901783972978592, | |
| "rewards/rejected": -0.10176645219326019, | |
| "sft_loss": 0.8286467790603638, | |
| "step": 1760 | |
| }, | |
| { | |
| "epoch": 2.9974597798475866, | |
| "grad_norm": 11.2490234375, | |
| "learning_rate": 0.0, | |
| "logits/chosen": -3.122014284133911, | |
| "logits/rejected": -3.142989158630371, | |
| "logps/chosen": -0.8531309962272644, | |
| "logps/rejected": -1.0983434915542603, | |
| "loss": 0.9206, | |
| "odds_ratio_loss": 0.6742203235626221, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.0853130966424942, | |
| "rewards/margins": 0.024521255865693092, | |
| "rewards/rejected": -0.10983435064554214, | |
| "sft_loss": 0.8531309962272644, | |
| "step": 1770 | |
| }, | |
| { | |
| "epoch": 2.9974597798475866, | |
| "step": 1770, | |
| "total_flos": 2.0399855839629804e+18, | |
| "train_loss": 0.8663494454938813, | |
| "train_runtime": 17638.0542, | |
| "train_samples_per_second": 1.607, | |
| "train_steps_per_second": 0.1 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 1770, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "total_flos": 2.0399855839629804e+18, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |