| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 1.0, | |
| "eval_steps": 500, | |
| "global_step": 73, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.01, | |
| "grad_norm": 31.510919537856985, | |
| "learning_rate": 6.25e-08, | |
| "logps/chosen": -49.24467849731445, | |
| "logps/rejected": -39.578514099121094, | |
| "loss": 0.7008, | |
| "losses/dpo": 0.6596390604972839, | |
| "losses/sft": 0.17909570038318634, | |
| "losses/total": 0.6596390604972839, | |
| "ref_logps/chosen": -49.112640380859375, | |
| "ref_logps/rejected": -39.590065002441406, | |
| "rewards/accuracies": 0.4296875, | |
| "rewards/chosen": -0.013204257935285568, | |
| "rewards/margins": -0.01435948722064495, | |
| "rewards/rejected": 0.001155228354036808, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.03, | |
| "grad_norm": 50.71439940508435, | |
| "learning_rate": 1.25e-07, | |
| "logps/chosen": -53.134193420410156, | |
| "logps/rejected": -47.03993225097656, | |
| "loss": 0.6899, | |
| "losses/dpo": 0.6913646459579468, | |
| "losses/sft": 0.1710137277841568, | |
| "losses/total": 0.6913646459579468, | |
| "ref_logps/chosen": -53.1795768737793, | |
| "ref_logps/rejected": -47.00617599487305, | |
| "rewards/accuracies": 0.5390625, | |
| "rewards/chosen": 0.004538209177553654, | |
| "rewards/margins": 0.007913690991699696, | |
| "rewards/rejected": -0.0033754808828234673, | |
| "step": 2 | |
| }, | |
| { | |
| "epoch": 0.04, | |
| "grad_norm": 42.035900231625114, | |
| "learning_rate": 1.875e-07, | |
| "logps/chosen": -51.67204666137695, | |
| "logps/rejected": -42.43119812011719, | |
| "loss": 0.6908, | |
| "losses/dpo": 0.6752724647521973, | |
| "losses/sft": 0.1361423283815384, | |
| "losses/total": 0.6752724647521973, | |
| "ref_logps/chosen": -51.709320068359375, | |
| "ref_logps/rejected": -42.41019058227539, | |
| "rewards/accuracies": 0.5390625, | |
| "rewards/chosen": 0.0037269049789756536, | |
| "rewards/margins": 0.00582832982763648, | |
| "rewards/rejected": -0.002101423917338252, | |
| "step": 3 | |
| }, | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 37.529489917527485, | |
| "learning_rate": 2.5e-07, | |
| "logps/chosen": -49.27662658691406, | |
| "logps/rejected": -42.45417404174805, | |
| "loss": 0.6949, | |
| "losses/dpo": 0.6901852488517761, | |
| "losses/sft": 0.18635118007659912, | |
| "losses/total": 0.6901852488517761, | |
| "ref_logps/chosen": -49.26988220214844, | |
| "ref_logps/rejected": -42.47301483154297, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.0006745874416083097, | |
| "rewards/margins": -0.0025582504458725452, | |
| "rewards/rejected": 0.0018836627714335918, | |
| "step": 4 | |
| }, | |
| { | |
| "epoch": 0.07, | |
| "grad_norm": 33.99395990426859, | |
| "learning_rate": 3.1249999999999997e-07, | |
| "logps/chosen": -48.10963439941406, | |
| "logps/rejected": -40.53562927246094, | |
| "loss": 0.693, | |
| "losses/dpo": 0.6474289894104004, | |
| "losses/sft": 0.1237928494811058, | |
| "losses/total": 0.6474289894104004, | |
| "ref_logps/chosen": -48.120365142822266, | |
| "ref_logps/rejected": -40.53252410888672, | |
| "rewards/accuracies": 0.4921875, | |
| "rewards/chosen": 0.001073037856258452, | |
| "rewards/margins": 0.001383528346195817, | |
| "rewards/rejected": -0.00031049124663695693, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.08, | |
| "grad_norm": 33.57616672063881, | |
| "learning_rate": 3.75e-07, | |
| "logps/chosen": -50.0238037109375, | |
| "logps/rejected": -41.26359558105469, | |
| "loss": 0.6976, | |
| "losses/dpo": 0.6670458316802979, | |
| "losses/sft": 0.15494827926158905, | |
| "losses/total": 0.6670458316802979, | |
| "ref_logps/chosen": -49.96715545654297, | |
| "ref_logps/rejected": -41.28278732299805, | |
| "rewards/accuracies": 0.5, | |
| "rewards/chosen": -0.005665063392370939, | |
| "rewards/margins": -0.007583992555737495, | |
| "rewards/rejected": 0.0019189291633665562, | |
| "step": 6 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 32.82497056854739, | |
| "learning_rate": 4.375e-07, | |
| "logps/chosen": -51.68867111206055, | |
| "logps/rejected": -43.37660217285156, | |
| "loss": 0.6921, | |
| "losses/dpo": 0.6866487264633179, | |
| "losses/sft": 0.13379694521427155, | |
| "losses/total": 0.6866487264633179, | |
| "ref_logps/chosen": -51.77002716064453, | |
| "ref_logps/rejected": -43.42585754394531, | |
| "rewards/accuracies": 0.515625, | |
| "rewards/chosen": 0.008136047050356865, | |
| "rewards/margins": 0.0032104291021823883, | |
| "rewards/rejected": 0.004925617948174477, | |
| "step": 7 | |
| }, | |
| { | |
| "epoch": 0.11, | |
| "grad_norm": 38.79691230760384, | |
| "learning_rate": 5e-07, | |
| "logps/chosen": -49.79241180419922, | |
| "logps/rejected": -42.46006774902344, | |
| "loss": 0.695, | |
| "losses/dpo": 0.6527105569839478, | |
| "losses/sft": 0.15517215430736542, | |
| "losses/total": 0.6527105569839478, | |
| "ref_logps/chosen": -49.76615905761719, | |
| "ref_logps/rejected": -42.45863342285156, | |
| "rewards/accuracies": 0.4765625, | |
| "rewards/chosen": -0.0026245731860399246, | |
| "rewards/margins": -0.002481258474290371, | |
| "rewards/rejected": -0.00014331366401165724, | |
| "step": 8 | |
| }, | |
| { | |
| "epoch": 0.12, | |
| "grad_norm": 226.56518786863586, | |
| "learning_rate": 4.997080567080816e-07, | |
| "logps/chosen": -54.879547119140625, | |
| "logps/rejected": -42.98774719238281, | |
| "loss": 0.6853, | |
| "losses/dpo": 0.6064831614494324, | |
| "losses/sft": 0.2268030345439911, | |
| "losses/total": 0.6064831614494324, | |
| "ref_logps/chosen": -54.92854309082031, | |
| "ref_logps/rejected": -42.86476135253906, | |
| "rewards/accuracies": 0.5546875, | |
| "rewards/chosen": 0.004900116473436356, | |
| "rewards/margins": 0.017198245972394943, | |
| "rewards/rejected": -0.012298129498958588, | |
| "step": 9 | |
| }, | |
| { | |
| "epoch": 0.14, | |
| "grad_norm": 50.10962021770019, | |
| "learning_rate": 4.988329086794122e-07, | |
| "logps/chosen": -52.08982467651367, | |
| "logps/rejected": -43.10820388793945, | |
| "loss": 0.6755, | |
| "losses/dpo": 0.7082348465919495, | |
| "losses/sft": 0.1494147777557373, | |
| "losses/total": 0.7082348465919495, | |
| "ref_logps/chosen": -52.199188232421875, | |
| "ref_logps/rejected": -42.8469123840332, | |
| "rewards/accuracies": 0.7109375, | |
| "rewards/chosen": 0.010936222970485687, | |
| "rewards/margins": 0.0370652973651886, | |
| "rewards/rejected": -0.02612907439470291, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 50.96166024464587, | |
| "learning_rate": 4.973765998627628e-07, | |
| "logps/chosen": -47.0179443359375, | |
| "logps/rejected": -42.876380920410156, | |
| "loss": 0.6764, | |
| "losses/dpo": 0.7454954385757446, | |
| "losses/sft": 0.11485376209020615, | |
| "losses/total": 0.7454954385757446, | |
| "ref_logps/chosen": -47.048309326171875, | |
| "ref_logps/rejected": -42.54939651489258, | |
| "rewards/accuracies": 0.671875, | |
| "rewards/chosen": 0.003036007285118103, | |
| "rewards/margins": 0.03573445603251457, | |
| "rewards/rejected": -0.03269844502210617, | |
| "step": 11 | |
| }, | |
| { | |
| "epoch": 0.16, | |
| "grad_norm": 43.95886877722781, | |
| "learning_rate": 4.953425315348533e-07, | |
| "logps/chosen": -47.39234161376953, | |
| "logps/rejected": -42.44198226928711, | |
| "loss": 0.6802, | |
| "losses/dpo": 0.6931471824645996, | |
| "losses/sft": 0.1578434258699417, | |
| "losses/total": 0.6931471824645996, | |
| "ref_logps/chosen": -47.38432312011719, | |
| "ref_logps/rejected": -42.15151596069336, | |
| "rewards/accuracies": 0.5859375, | |
| "rewards/chosen": -0.0008019641973078251, | |
| "rewards/margins": 0.028244582936167717, | |
| "rewards/rejected": -0.029046546667814255, | |
| "step": 12 | |
| }, | |
| { | |
| "epoch": 0.18, | |
| "grad_norm": 35.92010395880078, | |
| "learning_rate": 4.92735454356513e-07, | |
| "logps/chosen": -47.71138381958008, | |
| "logps/rejected": -40.651554107666016, | |
| "loss": 0.6671, | |
| "losses/dpo": 0.6601188778877258, | |
| "losses/sft": 0.12200119346380234, | |
| "losses/total": 0.6601188778877258, | |
| "ref_logps/chosen": -47.742637634277344, | |
| "ref_logps/rejected": -40.12622833251953, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": 0.003125616116449237, | |
| "rewards/margins": 0.055658161640167236, | |
| "rewards/rejected": -0.052532538771629333, | |
| "step": 13 | |
| }, | |
| { | |
| "epoch": 0.19, | |
| "grad_norm": 38.42448037262164, | |
| "learning_rate": 4.895614572772916e-07, | |
| "logps/chosen": -49.50225830078125, | |
| "logps/rejected": -41.507652282714844, | |
| "loss": 0.6657, | |
| "losses/dpo": 0.7294200658798218, | |
| "losses/sft": 0.15328460931777954, | |
| "losses/total": 0.7294200658798218, | |
| "ref_logps/chosen": -49.43043518066406, | |
| "ref_logps/rejected": -40.846160888671875, | |
| "rewards/accuracies": 0.6953125, | |
| "rewards/chosen": -0.007182045839726925, | |
| "rewards/margins": 0.058967407792806625, | |
| "rewards/rejected": -0.06614945828914642, | |
| "step": 14 | |
| }, | |
| { | |
| "epoch": 0.21, | |
| "grad_norm": 35.51673879153569, | |
| "learning_rate": 4.858279533144357e-07, | |
| "logps/chosen": -50.58116149902344, | |
| "logps/rejected": -44.15226745605469, | |
| "loss": 0.6648, | |
| "losses/dpo": 0.7195499539375305, | |
| "losses/sft": 0.12852783501148224, | |
| "losses/total": 0.7195499539375305, | |
| "ref_logps/chosen": -50.35996627807617, | |
| "ref_logps/rejected": -43.29161071777344, | |
| "rewards/accuracies": 0.671875, | |
| "rewards/chosen": -0.02211933210492134, | |
| "rewards/margins": 0.06394598633050919, | |
| "rewards/rejected": -0.08606532216072083, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.22, | |
| "grad_norm": 34.20405799205637, | |
| "learning_rate": 4.815436622394441e-07, | |
| "logps/chosen": -53.629356384277344, | |
| "logps/rejected": -45.9289665222168, | |
| "loss": 0.6455, | |
| "losses/dpo": 0.6931471824645996, | |
| "losses/sft": 0.14978539943695068, | |
| "losses/total": 0.6931471824645996, | |
| "ref_logps/chosen": -53.546119689941406, | |
| "ref_logps/rejected": -44.75611114501953, | |
| "rewards/accuracies": 0.7109375, | |
| "rewards/chosen": -0.00832361076027155, | |
| "rewards/margins": 0.10896164178848267, | |
| "rewards/rejected": -0.11728526651859283, | |
| "step": 16 | |
| }, | |
| { | |
| "epoch": 0.23, | |
| "grad_norm": 51.47866505976329, | |
| "learning_rate": 4.767185902126363e-07, | |
| "logps/chosen": -52.55246353149414, | |
| "logps/rejected": -48.530906677246094, | |
| "loss": 0.6445, | |
| "losses/dpo": 0.6025716662406921, | |
| "losses/sft": 0.17564229667186737, | |
| "losses/total": 0.6025716662406921, | |
| "ref_logps/chosen": -52.30644989013672, | |
| "ref_logps/rejected": -47.189701080322266, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -0.02460121549665928, | |
| "rewards/margins": 0.10951894521713257, | |
| "rewards/rejected": -0.1341201663017273, | |
| "step": 17 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 42.49951077408584, | |
| "learning_rate": 4.7136400641330245e-07, | |
| "logps/chosen": -52.193199157714844, | |
| "logps/rejected": -43.489959716796875, | |
| "loss": 0.6225, | |
| "losses/dpo": 0.4434754252433777, | |
| "losses/sft": 0.13364653289318085, | |
| "losses/total": 0.4434754252433777, | |
| "ref_logps/chosen": -51.82938003540039, | |
| "ref_logps/rejected": -41.477630615234375, | |
| "rewards/accuracies": 0.7265625, | |
| "rewards/chosen": -0.03638196364045143, | |
| "rewards/margins": 0.16485059261322021, | |
| "rewards/rejected": -0.20123255252838135, | |
| "step": 18 | |
| }, | |
| { | |
| "epoch": 0.26, | |
| "grad_norm": 34.30863345836372, | |
| "learning_rate": 4.6549241672001225e-07, | |
| "logps/chosen": -49.611228942871094, | |
| "logps/rejected": -46.756103515625, | |
| "loss": 0.6071, | |
| "losses/dpo": 0.6045699715614319, | |
| "losses/sft": 0.13909928500652313, | |
| "losses/total": 0.6045699715614319, | |
| "ref_logps/chosen": -49.14532470703125, | |
| "ref_logps/rejected": -44.29924774169922, | |
| "rewards/accuracies": 0.7890625, | |
| "rewards/chosen": -0.046590566635131836, | |
| "rewards/margins": 0.19909515976905823, | |
| "rewards/rejected": -0.24568572640419006, | |
| "step": 19 | |
| }, | |
| { | |
| "epoch": 0.27, | |
| "grad_norm": 35.537459009303014, | |
| "learning_rate": 4.591175345025566e-07, | |
| "logps/chosen": -50.094032287597656, | |
| "logps/rejected": -46.121707916259766, | |
| "loss": 0.6058, | |
| "losses/dpo": 0.5356044173240662, | |
| "losses/sft": 0.1512974053621292, | |
| "losses/total": 0.5356044173240662, | |
| "ref_logps/chosen": -49.32575607299805, | |
| "ref_logps/rejected": -43.3276481628418, | |
| "rewards/accuracies": 0.7578125, | |
| "rewards/chosen": -0.07682754099369049, | |
| "rewards/margins": 0.20257848501205444, | |
| "rewards/rejected": -0.27940604090690613, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.29, | |
| "grad_norm": 82.79188209516856, | |
| "learning_rate": 4.5225424859373684e-07, | |
| "logps/chosen": -53.73488235473633, | |
| "logps/rejected": -48.036128997802734, | |
| "loss": 0.5936, | |
| "losses/dpo": 0.6363024711608887, | |
| "losses/sft": 0.18370339274406433, | |
| "losses/total": 0.6363024711608887, | |
| "ref_logps/chosen": -52.835113525390625, | |
| "ref_logps/rejected": -44.653133392333984, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -0.08997656404972076, | |
| "rewards/margins": 0.24832308292388916, | |
| "rewards/rejected": -0.33829963207244873, | |
| "step": 21 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "grad_norm": 35.68500953981497, | |
| "learning_rate": 4.4491858851580553e-07, | |
| "logps/chosen": -52.28797149658203, | |
| "logps/rejected": -48.59185791015625, | |
| "loss": 0.5956, | |
| "losses/dpo": 0.6950743198394775, | |
| "losses/sft": 0.3061237335205078, | |
| "losses/total": 0.6950743198394775, | |
| "ref_logps/chosen": -50.30400085449219, | |
| "ref_logps/rejected": -43.93888854980469, | |
| "rewards/accuracies": 0.6953125, | |
| "rewards/chosen": -0.19839699566364288, | |
| "rewards/margins": 0.2669002413749695, | |
| "rewards/rejected": -0.46529728174209595, | |
| "step": 22 | |
| }, | |
| { | |
| "epoch": 0.32, | |
| "grad_norm": 51.2013048213769, | |
| "learning_rate": 4.3712768704277524e-07, | |
| "logps/chosen": -50.60584259033203, | |
| "logps/rejected": -49.11386489868164, | |
| "loss": 0.5413, | |
| "losses/dpo": 1.0638338327407837, | |
| "losses/sft": 0.1654026359319687, | |
| "losses/total": 1.0638338327407837, | |
| "ref_logps/chosen": -48.48411560058594, | |
| "ref_logps/rejected": -42.86372375488281, | |
| "rewards/accuracies": 0.7890625, | |
| "rewards/chosen": -0.2121725082397461, | |
| "rewards/margins": 0.4128417372703552, | |
| "rewards/rejected": -0.6250141859054565, | |
| "step": 23 | |
| }, | |
| { | |
| "epoch": 0.33, | |
| "grad_norm": 26.152258770479694, | |
| "learning_rate": 4.2889974018603024e-07, | |
| "logps/chosen": -53.90440368652344, | |
| "logps/rejected": -50.59031295776367, | |
| "loss": 0.5421, | |
| "losses/dpo": 0.7262043952941895, | |
| "losses/sft": 0.18153545260429382, | |
| "losses/total": 0.7262043952941895, | |
| "ref_logps/chosen": -51.315208435058594, | |
| "ref_logps/rejected": -43.73783874511719, | |
| "rewards/accuracies": 0.734375, | |
| "rewards/chosen": -0.2589200735092163, | |
| "rewards/margins": 0.4263269305229187, | |
| "rewards/rejected": -0.6852469444274902, | |
| "step": 24 | |
| }, | |
| { | |
| "epoch": 0.34, | |
| "grad_norm": 49.33026538547341, | |
| "learning_rate": 4.2025396469669926e-07, | |
| "logps/chosen": -53.82769775390625, | |
| "logps/rejected": -50.70613098144531, | |
| "loss": 0.5517, | |
| "losses/dpo": 0.41891151666641235, | |
| "losses/sft": 0.1567077487707138, | |
| "losses/total": 0.41891151666641235, | |
| "ref_logps/chosen": -50.04922103881836, | |
| "ref_logps/rejected": -42.47876739501953, | |
| "rewards/accuracies": 0.7109375, | |
| "rewards/chosen": -0.37784749269485474, | |
| "rewards/margins": 0.4448884427547455, | |
| "rewards/rejected": -0.8227359056472778, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.36, | |
| "grad_norm": 38.54013817065724, | |
| "learning_rate": 4.112105531840426e-07, | |
| "logps/chosen": -56.83380889892578, | |
| "logps/rejected": -55.79877471923828, | |
| "loss": 0.5145, | |
| "losses/dpo": 0.25203394889831543, | |
| "losses/sft": 0.15007992088794708, | |
| "losses/total": 0.25203394889831543, | |
| "ref_logps/chosen": -51.92417907714844, | |
| "ref_logps/rejected": -45.255348205566406, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.49096283316612244, | |
| "rewards/margins": 0.5633795261383057, | |
| "rewards/rejected": -1.0543423891067505, | |
| "step": 26 | |
| }, | |
| { | |
| "epoch": 0.37, | |
| "grad_norm": 35.25775541354148, | |
| "learning_rate": 4.017906269546778e-07, | |
| "logps/chosen": -57.45189666748047, | |
| "logps/rejected": -53.96529769897461, | |
| "loss": 0.5329, | |
| "losses/dpo": 0.3053484857082367, | |
| "losses/sft": 0.12551482021808624, | |
| "losses/total": 0.3053484857082367, | |
| "ref_logps/chosen": -52.189697265625, | |
| "ref_logps/rejected": -44.003353118896484, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -0.5262198448181152, | |
| "rewards/margins": 0.46997466683387756, | |
| "rewards/rejected": -0.99619460105896, | |
| "step": 27 | |
| }, | |
| { | |
| "epoch": 0.38, | |
| "grad_norm": 23.64859158816546, | |
| "learning_rate": 3.920161866827889e-07, | |
| "logps/chosen": -57.529579162597656, | |
| "logps/rejected": -56.24253845214844, | |
| "loss": 0.4882, | |
| "losses/dpo": 0.67308509349823, | |
| "losses/sft": 0.12211936712265015, | |
| "losses/total": 0.67308509349823, | |
| "ref_logps/chosen": -51.290321350097656, | |
| "ref_logps/rejected": -43.70161437988281, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.6239261627197266, | |
| "rewards/margins": 0.6301661133766174, | |
| "rewards/rejected": -1.2540922164916992, | |
| "step": 28 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "grad_norm": 34.075060841718205, | |
| "learning_rate": 3.8191006102653317e-07, | |
| "logps/chosen": -54.39567947387695, | |
| "logps/rejected": -53.69304656982422, | |
| "loss": 0.515, | |
| "losses/dpo": 0.3636208176612854, | |
| "losses/sft": 0.15627621114253998, | |
| "losses/total": 0.3636208176612854, | |
| "ref_logps/chosen": -46.61610412597656, | |
| "ref_logps/rejected": -40.169898986816406, | |
| "rewards/accuracies": 0.7890625, | |
| "rewards/chosen": -0.7779572010040283, | |
| "rewards/margins": 0.5743571519851685, | |
| "rewards/rejected": -1.3523142337799072, | |
| "step": 29 | |
| }, | |
| { | |
| "epoch": 0.41, | |
| "grad_norm": 39.075544491508126, | |
| "learning_rate": 3.7149585331065145e-07, | |
| "logps/chosen": -55.852046966552734, | |
| "logps/rejected": -56.0112419128418, | |
| "loss": 0.4987, | |
| "losses/dpo": 0.9463083744049072, | |
| "losses/sft": 0.1410434991121292, | |
| "losses/total": 0.9463083744049072, | |
| "ref_logps/chosen": -48.19220733642578, | |
| "ref_logps/rejected": -42.009246826171875, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.765984058380127, | |
| "rewards/margins": 0.634215772151947, | |
| "rewards/rejected": -1.4001998901367188, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.42, | |
| "grad_norm": 27.598875905640913, | |
| "learning_rate": 3.6079788639981036e-07, | |
| "logps/chosen": -60.89286804199219, | |
| "logps/rejected": -58.91151428222656, | |
| "loss": 0.477, | |
| "losses/dpo": 0.5266605615615845, | |
| "losses/sft": 0.14658664166927338, | |
| "losses/total": 0.5266605615615845, | |
| "ref_logps/chosen": -51.61863327026367, | |
| "ref_logps/rejected": -42.28534698486328, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -0.9274235963821411, | |
| "rewards/margins": 0.7351932525634766, | |
| "rewards/rejected": -1.6626169681549072, | |
| "step": 31 | |
| }, | |
| { | |
| "epoch": 0.44, | |
| "grad_norm": 28.756497878941868, | |
| "learning_rate": 3.498411458914238e-07, | |
| "logps/chosen": -60.40979766845703, | |
| "logps/rejected": -60.34819793701172, | |
| "loss": 0.4847, | |
| "losses/dpo": 0.4884541630744934, | |
| "losses/sft": 0.19494454562664032, | |
| "losses/total": 0.4884541630744934, | |
| "ref_logps/chosen": -52.04460906982422, | |
| "ref_logps/rejected": -44.97260665893555, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -0.8365185260772705, | |
| "rewards/margins": 0.701040506362915, | |
| "rewards/rejected": -1.537559151649475, | |
| "step": 32 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "grad_norm": 28.434577214059978, | |
| "learning_rate": 3.3865122176063385e-07, | |
| "logps/chosen": -57.96968460083008, | |
| "logps/rejected": -56.83502960205078, | |
| "loss": 0.4677, | |
| "losses/dpo": 0.5375417470932007, | |
| "losses/sft": 0.148137629032135, | |
| "losses/total": 0.5375417470932007, | |
| "ref_logps/chosen": -47.51142120361328, | |
| "ref_logps/rejected": -38.619873046875, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": -1.0458261966705322, | |
| "rewards/margins": 0.7756892442703247, | |
| "rewards/rejected": -1.8215153217315674, | |
| "step": 33 | |
| }, | |
| { | |
| "epoch": 0.47, | |
| "grad_norm": 29.49134519608509, | |
| "learning_rate": 3.272542485937368e-07, | |
| "logps/chosen": -62.14280700683594, | |
| "logps/rejected": -61.54853820800781, | |
| "loss": 0.4961, | |
| "losses/dpo": 0.7278311848640442, | |
| "losses/sft": 0.1993408501148224, | |
| "losses/total": 0.7278311848640442, | |
| "ref_logps/chosen": -51.82103729248047, | |
| "ref_logps/rejected": -44.025291442871094, | |
| "rewards/accuracies": 0.75, | |
| "rewards/chosen": -1.0321770906448364, | |
| "rewards/margins": 0.7201482653617859, | |
| "rewards/rejected": -1.7523252964019775, | |
| "step": 34 | |
| }, | |
| { | |
| "epoch": 0.48, | |
| "grad_norm": 26.468466564434074, | |
| "learning_rate": 3.1567684454964674e-07, | |
| "logps/chosen": -56.682411193847656, | |
| "logps/rejected": -58.0302734375, | |
| "loss": 0.4396, | |
| "losses/dpo": 0.5649744868278503, | |
| "losses/sft": 0.129312202334404, | |
| "losses/total": 0.5649744868278503, | |
| "ref_logps/chosen": -47.770896911621094, | |
| "ref_logps/rejected": -40.46588897705078, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -0.8911517858505249, | |
| "rewards/margins": 0.8652870059013367, | |
| "rewards/rejected": -1.7564387321472168, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.49, | |
| "grad_norm": 25.595938355510107, | |
| "learning_rate": 3.0394604919195157e-07, | |
| "logps/chosen": -61.79121398925781, | |
| "logps/rejected": -62.89993667602539, | |
| "loss": 0.4745, | |
| "losses/dpo": 0.2738940417766571, | |
| "losses/sft": 0.15838046371936798, | |
| "losses/total": 0.2738940417766571, | |
| "ref_logps/chosen": -49.21980285644531, | |
| "ref_logps/rejected": -42.15373992919922, | |
| "rewards/accuracies": 0.7578125, | |
| "rewards/chosen": -1.257140874862671, | |
| "rewards/margins": 0.8174787759780884, | |
| "rewards/rejected": -2.074619770050049, | |
| "step": 36 | |
| }, | |
| { | |
| "epoch": 0.51, | |
| "grad_norm": 24.43093018755541, | |
| "learning_rate": 2.920892603367596e-07, | |
| "logps/chosen": -60.27898406982422, | |
| "logps/rejected": -61.00114822387695, | |
| "loss": 0.4791, | |
| "losses/dpo": 0.5097879767417908, | |
| "losses/sft": 0.17959001660346985, | |
| "losses/total": 0.5097879767417908, | |
| "ref_logps/chosen": -49.49871826171875, | |
| "ref_logps/rejected": -42.78453063964844, | |
| "rewards/accuracies": 0.7890625, | |
| "rewards/chosen": -1.078026533126831, | |
| "rewards/margins": 0.7436352372169495, | |
| "rewards/rejected": -1.8216617107391357, | |
| "step": 37 | |
| }, | |
| { | |
| "epoch": 0.52, | |
| "grad_norm": 27.08759050745191, | |
| "learning_rate": 2.801341700638307e-07, | |
| "logps/chosen": -62.44047164916992, | |
| "logps/rejected": -62.224884033203125, | |
| "loss": 0.4767, | |
| "losses/dpo": 0.37803658843040466, | |
| "losses/sft": 0.18652431666851044, | |
| "losses/total": 0.37803658843040466, | |
| "ref_logps/chosen": -50.557559967041016, | |
| "ref_logps/rejected": -41.98173522949219, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -1.1882915496826172, | |
| "rewards/margins": 0.8360234498977661, | |
| "rewards/rejected": -2.0243148803710938, | |
| "step": 38 | |
| }, | |
| { | |
| "epoch": 0.53, | |
| "grad_norm": 33.43704122805882, | |
| "learning_rate": 2.681087000404406e-07, | |
| "logps/chosen": -65.03423309326172, | |
| "logps/rejected": -65.04768371582031, | |
| "loss": 0.5084, | |
| "losses/dpo": 0.7027170658111572, | |
| "losses/sft": 0.12946006655693054, | |
| "losses/total": 0.7027170658111572, | |
| "ref_logps/chosen": -51.817020416259766, | |
| "ref_logps/rejected": -44.10376739501953, | |
| "rewards/accuracies": 0.71875, | |
| "rewards/chosen": -1.3217215538024902, | |
| "rewards/margins": 0.7726694345474243, | |
| "rewards/rejected": -2.094391107559204, | |
| "step": 39 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "grad_norm": 129.45570096870608, | |
| "learning_rate": 2.5604093630903305e-07, | |
| "logps/chosen": -59.94195556640625, | |
| "logps/rejected": -63.19525909423828, | |
| "loss": 0.4527, | |
| "losses/dpo": 0.7064068913459778, | |
| "losses/sft": 0.13724827766418457, | |
| "losses/total": 0.7064068913459778, | |
| "ref_logps/chosen": -47.65526580810547, | |
| "ref_logps/rejected": -42.33349609375, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": -1.2286689281463623, | |
| "rewards/margins": 0.8575077056884766, | |
| "rewards/rejected": -2.086176872253418, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.56, | |
| "grad_norm": 24.012368893482737, | |
| "learning_rate": 2.43959063690967e-07, | |
| "logps/chosen": -61.86077880859375, | |
| "logps/rejected": -66.92259979248047, | |
| "loss": 0.4553, | |
| "losses/dpo": 0.3154614567756653, | |
| "losses/sft": 0.16294528543949127, | |
| "losses/total": 0.3154614567756653, | |
| "ref_logps/chosen": -48.4053955078125, | |
| "ref_logps/rejected": -44.16739273071289, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": -1.3455381393432617, | |
| "rewards/margins": 0.9299829006195068, | |
| "rewards/rejected": -2.2755210399627686, | |
| "step": 41 | |
| }, | |
| { | |
| "epoch": 0.58, | |
| "grad_norm": 102.16273879292123, | |
| "learning_rate": 2.3189129995955942e-07, | |
| "logps/chosen": -64.28011322021484, | |
| "logps/rejected": -64.86600494384766, | |
| "loss": 0.4642, | |
| "losses/dpo": 0.4425658583641052, | |
| "losses/sft": 0.11157938092947006, | |
| "losses/total": 0.4425658583641052, | |
| "ref_logps/chosen": -53.148860931396484, | |
| "ref_logps/rejected": -45.69209289550781, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -1.113125205039978, | |
| "rewards/margins": 0.8042662143707275, | |
| "rewards/rejected": -1.9173914194107056, | |
| "step": 42 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "grad_norm": 27.45890404880792, | |
| "learning_rate": 2.1986582993616925e-07, | |
| "logps/chosen": -64.73760223388672, | |
| "logps/rejected": -63.79928207397461, | |
| "loss": 0.4607, | |
| "losses/dpo": 0.08850211650133133, | |
| "losses/sft": 0.1770356148481369, | |
| "losses/total": 0.08850211650133133, | |
| "ref_logps/chosen": -52.7093391418457, | |
| "ref_logps/rejected": -42.18277359008789, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -1.2028264999389648, | |
| "rewards/margins": 0.9588243961334229, | |
| "rewards/rejected": -2.1616508960723877, | |
| "step": 43 | |
| }, | |
| { | |
| "epoch": 0.6, | |
| "grad_norm": 35.977504794321945, | |
| "learning_rate": 2.0791073966324034e-07, | |
| "logps/chosen": -65.26310729980469, | |
| "logps/rejected": -66.70973205566406, | |
| "loss": 0.4531, | |
| "losses/dpo": 0.7271055579185486, | |
| "losses/sft": 0.1396849900484085, | |
| "losses/total": 0.7271055579185486, | |
| "ref_logps/chosen": -53.6609001159668, | |
| "ref_logps/rejected": -46.01152801513672, | |
| "rewards/accuracies": 0.8515625, | |
| "rewards/chosen": -1.1602208614349365, | |
| "rewards/margins": 0.909599781036377, | |
| "rewards/rejected": -2.0698208808898926, | |
| "step": 44 | |
| }, | |
| { | |
| "epoch": 0.62, | |
| "grad_norm": 30.8655762680209, | |
| "learning_rate": 1.960539508080485e-07, | |
| "logps/chosen": -62.69697952270508, | |
| "logps/rejected": -64.87743377685547, | |
| "loss": 0.4482, | |
| "losses/dpo": 0.7160241603851318, | |
| "losses/sft": 0.16245654225349426, | |
| "losses/total": 0.7160241603851318, | |
| "ref_logps/chosen": -49.983802795410156, | |
| "ref_logps/rejected": -43.17237091064453, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": -1.2713178396224976, | |
| "rewards/margins": 0.8991883993148804, | |
| "rewards/rejected": -2.170506238937378, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.63, | |
| "grad_norm": 42.203067350847334, | |
| "learning_rate": 1.8432315545035327e-07, | |
| "logps/chosen": -65.14309692382812, | |
| "logps/rejected": -64.79584503173828, | |
| "loss": 0.4638, | |
| "losses/dpo": 0.647499144077301, | |
| "losses/sft": 0.17309847474098206, | |
| "losses/total": 0.647499144077301, | |
| "ref_logps/chosen": -51.99322509765625, | |
| "ref_logps/rejected": -42.78443145751953, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -1.3149867057800293, | |
| "rewards/margins": 0.8861545324325562, | |
| "rewards/rejected": -2.201141357421875, | |
| "step": 46 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "grad_norm": 26.70333995315738, | |
| "learning_rate": 1.7274575140626315e-07, | |
| "logps/chosen": -64.62788391113281, | |
| "logps/rejected": -66.00929260253906, | |
| "loss": 0.3755, | |
| "losses/dpo": 0.7572497129440308, | |
| "losses/sft": 0.16420619189739227, | |
| "losses/total": 0.7572497129440308, | |
| "ref_logps/chosen": -53.670265197753906, | |
| "ref_logps/rejected": -43.973365783691406, | |
| "rewards/accuracies": 0.8671875, | |
| "rewards/chosen": -1.0957624912261963, | |
| "rewards/margins": 1.1078301668167114, | |
| "rewards/rejected": -2.203592538833618, | |
| "step": 47 | |
| }, | |
| { | |
| "epoch": 0.66, | |
| "grad_norm": 30.007336064883145, | |
| "learning_rate": 1.6134877823936607e-07, | |
| "logps/chosen": -60.07398223876953, | |
| "logps/rejected": -63.938331604003906, | |
| "loss": 0.4222, | |
| "losses/dpo": 0.4377414286136627, | |
| "losses/sft": 0.11289513111114502, | |
| "losses/total": 0.4377414286136627, | |
| "ref_logps/chosen": -48.226722717285156, | |
| "ref_logps/rejected": -42.14054870605469, | |
| "rewards/accuracies": 0.765625, | |
| "rewards/chosen": -1.1847256422042847, | |
| "rewards/margins": 0.995052695274353, | |
| "rewards/rejected": -2.1797783374786377, | |
| "step": 48 | |
| }, | |
| { | |
| "epoch": 0.67, | |
| "grad_norm": 33.18269229637362, | |
| "learning_rate": 1.5015885410857614e-07, | |
| "logps/chosen": -63.292476654052734, | |
| "logps/rejected": -66.98123168945312, | |
| "loss": 0.4246, | |
| "losses/dpo": 0.3581680655479431, | |
| "losses/sft": 0.17332157492637634, | |
| "losses/total": 0.3581680655479431, | |
| "ref_logps/chosen": -50.916664123535156, | |
| "ref_logps/rejected": -45.03084182739258, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -1.2375812530517578, | |
| "rewards/margins": 0.9574571847915649, | |
| "rewards/rejected": -2.195038318634033, | |
| "step": 49 | |
| }, | |
| { | |
| "epoch": 0.68, | |
| "grad_norm": 32.535671402481796, | |
| "learning_rate": 1.392021136001897e-07, | |
| "logps/chosen": -61.653507232666016, | |
| "logps/rejected": -66.69647979736328, | |
| "loss": 0.3784, | |
| "losses/dpo": 0.14524562656879425, | |
| "losses/sft": 0.1722523272037506, | |
| "losses/total": 0.14524562656879425, | |
| "ref_logps/chosen": -51.076236724853516, | |
| "ref_logps/rejected": -44.246944427490234, | |
| "rewards/accuracies": 0.8046875, | |
| "rewards/chosen": -1.0577270984649658, | |
| "rewards/margins": 1.1872267723083496, | |
| "rewards/rejected": -2.2449538707733154, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.7, | |
| "grad_norm": 25.494002931399418, | |
| "learning_rate": 1.2850414668934847e-07, | |
| "logps/chosen": -63.76778030395508, | |
| "logps/rejected": -66.36669921875, | |
| "loss": 0.4239, | |
| "losses/dpo": 0.5600922107696533, | |
| "losses/sft": 0.13496388494968414, | |
| "losses/total": 0.5600922107696533, | |
| "ref_logps/chosen": -49.28026580810547, | |
| "ref_logps/rejected": -41.501556396484375, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -1.4487512111663818, | |
| "rewards/margins": 1.0377633571624756, | |
| "rewards/rejected": -2.4865145683288574, | |
| "step": 51 | |
| }, | |
| { | |
| "epoch": 0.71, | |
| "grad_norm": 24.257143077705184, | |
| "learning_rate": 1.1808993897346678e-07, | |
| "logps/chosen": -60.98112487792969, | |
| "logps/rejected": -66.6905517578125, | |
| "loss": 0.4463, | |
| "losses/dpo": 0.6072797775268555, | |
| "losses/sft": 0.14662015438079834, | |
| "losses/total": 0.6072797775268555, | |
| "ref_logps/chosen": -48.362701416015625, | |
| "ref_logps/rejected": -44.13787841796875, | |
| "rewards/accuracies": 0.7734375, | |
| "rewards/chosen": -1.2618422508239746, | |
| "rewards/margins": 0.9934254288673401, | |
| "rewards/rejected": -2.25526762008667, | |
| "step": 52 | |
| }, | |
| { | |
| "epoch": 0.73, | |
| "grad_norm": 28.99400991059714, | |
| "learning_rate": 1.0798381331721107e-07, | |
| "logps/chosen": -61.79584884643555, | |
| "logps/rejected": -64.41800689697266, | |
| "loss": 0.4073, | |
| "losses/dpo": 0.17208106815814972, | |
| "losses/sft": 0.14093177020549774, | |
| "losses/total": 0.17208106815814972, | |
| "ref_logps/chosen": -49.47767639160156, | |
| "ref_logps/rejected": -41.640419006347656, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -1.2318172454833984, | |
| "rewards/margins": 1.0459411144256592, | |
| "rewards/rejected": -2.2777583599090576, | |
| "step": 53 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "grad_norm": 32.15451904929163, | |
| "learning_rate": 9.82093730453222e-08, | |
| "logps/chosen": -62.59501647949219, | |
| "logps/rejected": -63.971126556396484, | |
| "loss": 0.5058, | |
| "losses/dpo": 0.3638117015361786, | |
| "losses/sft": 0.14573359489440918, | |
| "losses/total": 0.3638117015361786, | |
| "ref_logps/chosen": -51.01821517944336, | |
| "ref_logps/rejected": -43.79859161376953, | |
| "rewards/accuracies": 0.7421875, | |
| "rewards/chosen": -1.157679796218872, | |
| "rewards/margins": 0.8595736026763916, | |
| "rewards/rejected": -2.0172533988952637, | |
| "step": 54 | |
| }, | |
| { | |
| "epoch": 0.75, | |
| "grad_norm": 29.71547196346147, | |
| "learning_rate": 8.87894468159574e-08, | |
| "logps/chosen": -63.908302307128906, | |
| "logps/rejected": -65.4727554321289, | |
| "loss": 0.4043, | |
| "losses/dpo": 1.4689037799835205, | |
| "losses/sft": 0.19870857894420624, | |
| "losses/total": 1.4689037799835205, | |
| "ref_logps/chosen": -49.52888870239258, | |
| "ref_logps/rejected": -39.608917236328125, | |
| "rewards/accuracies": 0.8203125, | |
| "rewards/chosen": -1.437941312789917, | |
| "rewards/margins": 1.148442029953003, | |
| "rewards/rejected": -2.586383581161499, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.77, | |
| "grad_norm": 40.070212554436814, | |
| "learning_rate": 7.974603530330067e-08, | |
| "logps/chosen": -63.663780212402344, | |
| "logps/rejected": -65.04582977294922, | |
| "loss": 0.4551, | |
| "losses/dpo": 0.7427124977111816, | |
| "losses/sft": 0.1415950208902359, | |
| "losses/total": 0.7427124977111816, | |
| "ref_logps/chosen": -52.447853088378906, | |
| "ref_logps/rejected": -44.403289794921875, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -1.1215929985046387, | |
| "rewards/margins": 0.9426611661911011, | |
| "rewards/rejected": -2.0642542839050293, | |
| "step": 56 | |
| }, | |
| { | |
| "epoch": 0.78, | |
| "grad_norm": 50.773025060467745, | |
| "learning_rate": 7.110025981396975e-08, | |
| "logps/chosen": -64.57611083984375, | |
| "logps/rejected": -67.85655212402344, | |
| "loss": 0.4677, | |
| "losses/dpo": 0.7480742335319519, | |
| "losses/sft": 0.1996467560529709, | |
| "losses/total": 0.7480742335319519, | |
| "ref_logps/chosen": -49.66741943359375, | |
| "ref_logps/rejected": -43.6092414855957, | |
| "rewards/accuracies": 0.796875, | |
| "rewards/chosen": -1.4908695220947266, | |
| "rewards/margins": 0.9338614344596863, | |
| "rewards/rejected": -2.4247307777404785, | |
| "step": 57 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "grad_norm": 151.98374859438076, | |
| "learning_rate": 6.28723129572247e-08, | |
| "logps/chosen": -63.19204330444336, | |
| "logps/rejected": -67.11865234375, | |
| "loss": 0.4125, | |
| "losses/dpo": 0.191142737865448, | |
| "losses/sft": 0.1258397251367569, | |
| "losses/total": 0.191142737865448, | |
| "ref_logps/chosen": -49.3205680847168, | |
| "ref_logps/rejected": -42.308128356933594, | |
| "rewards/accuracies": 0.859375, | |
| "rewards/chosen": -1.387147307395935, | |
| "rewards/margins": 1.0939054489135742, | |
| "rewards/rejected": -2.481052875518799, | |
| "step": 58 | |
| }, | |
| { | |
| "epoch": 0.81, | |
| "grad_norm": 68.1040384525775, | |
| "learning_rate": 5.508141148419443e-08, | |
| "logps/chosen": -66.73199462890625, | |
| "logps/rejected": -66.16500854492188, | |
| "loss": 0.416, | |
| "losses/dpo": 0.4806699752807617, | |
| "losses/sft": 0.17691649496555328, | |
| "losses/total": 0.4806699752807617, | |
| "ref_logps/chosen": -52.95691680908203, | |
| "ref_logps/rejected": -42.309608459472656, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -1.3775070905685425, | |
| "rewards/margins": 1.0080331563949585, | |
| "rewards/rejected": -2.38554048538208, | |
| "step": 59 | |
| }, | |
| { | |
| "epoch": 0.82, | |
| "grad_norm": 40.930870088616274, | |
| "learning_rate": 4.774575140626316e-08, | |
| "logps/chosen": -65.13594055175781, | |
| "logps/rejected": -67.62340545654297, | |
| "loss": 0.4084, | |
| "losses/dpo": 0.3780416250228882, | |
| "losses/sft": 0.070173479616642, | |
| "losses/total": 0.3780416250228882, | |
| "ref_logps/chosen": -50.85914611816406, | |
| "ref_logps/rejected": -41.8093147277832, | |
| "rewards/accuracies": 0.828125, | |
| "rewards/chosen": -1.4276790618896484, | |
| "rewards/margins": 1.153730034828186, | |
| "rewards/rejected": -2.581409215927124, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "grad_norm": 30.260562657822298, | |
| "learning_rate": 4.0882465497443313e-08, | |
| "logps/chosen": -63.07252883911133, | |
| "logps/rejected": -69.03398132324219, | |
| "loss": 0.4369, | |
| "losses/dpo": 0.36678266525268555, | |
| "losses/sft": 0.12201398611068726, | |
| "losses/total": 0.36678266525268555, | |
| "ref_logps/chosen": -49.61937713623047, | |
| "ref_logps/rejected": -44.632652282714844, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -1.3453152179718018, | |
| "rewards/margins": 1.0948173999786377, | |
| "rewards/rejected": -2.4401326179504395, | |
| "step": 61 | |
| }, | |
| { | |
| "epoch": 0.85, | |
| "grad_norm": 24.402238672879346, | |
| "learning_rate": 3.450758327998768e-08, | |
| "logps/chosen": -63.65578842163086, | |
| "logps/rejected": -67.62722778320312, | |
| "loss": 0.3813, | |
| "losses/dpo": 0.7685420513153076, | |
| "losses/sft": 0.19448544085025787, | |
| "losses/total": 0.7685420513153076, | |
| "ref_logps/chosen": -51.653221130371094, | |
| "ref_logps/rejected": -43.50916290283203, | |
| "rewards/accuracies": 0.8359375, | |
| "rewards/chosen": -1.2002568244934082, | |
| "rewards/margins": 1.2115496397018433, | |
| "rewards/rejected": -2.411806583404541, | |
| "step": 62 | |
| }, | |
| { | |
| "epoch": 0.86, | |
| "grad_norm": 24.77840549005978, | |
| "learning_rate": 2.863599358669755e-08, | |
| "logps/chosen": -65.08932495117188, | |
| "logps/rejected": -66.61253356933594, | |
| "loss": 0.4291, | |
| "losses/dpo": 1.248164176940918, | |
| "losses/sft": 0.15433447062969208, | |
| "losses/total": 1.248164176940918, | |
| "ref_logps/chosen": -52.82990646362305, | |
| "ref_logps/rejected": -44.81475830078125, | |
| "rewards/accuracies": 0.8046875, | |
| "rewards/chosen": -1.2259418964385986, | |
| "rewards/margins": 0.9538350105285645, | |
| "rewards/rejected": -2.179776906967163, | |
| "step": 63 | |
| }, | |
| { | |
| "epoch": 0.88, | |
| "grad_norm": 37.580661698465605, | |
| "learning_rate": 2.3281409787363648e-08, | |
| "logps/chosen": -61.57237243652344, | |
| "logps/rejected": -65.20365142822266, | |
| "loss": 0.4063, | |
| "losses/dpo": 0.09678905457258224, | |
| "losses/sft": 0.15513929724693298, | |
| "losses/total": 0.09678905457258224, | |
| "ref_logps/chosen": -48.36906814575195, | |
| "ref_logps/rejected": -41.755638122558594, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -1.3203303813934326, | |
| "rewards/margins": 1.0244702100753784, | |
| "rewards/rejected": -2.3448004722595215, | |
| "step": 64 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "grad_norm": 52.86928073665909, | |
| "learning_rate": 1.845633776055591e-08, | |
| "logps/chosen": -63.794559478759766, | |
| "logps/rejected": -68.59591674804688, | |
| "loss": 0.4236, | |
| "losses/dpo": 0.5401971936225891, | |
| "losses/sft": 0.1492851972579956, | |
| "losses/total": 0.5401971936225891, | |
| "ref_logps/chosen": -51.490814208984375, | |
| "ref_logps/rejected": -46.14380645751953, | |
| "rewards/accuracies": 0.8125, | |
| "rewards/chosen": -1.2303749322891235, | |
| "rewards/margins": 1.0148357152938843, | |
| "rewards/rejected": -2.245210647583008, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.9, | |
| "grad_norm": 26.186736903994795, | |
| "learning_rate": 1.4172046685564209e-08, | |
| "logps/chosen": -63.65757751464844, | |
| "logps/rejected": -66.78644561767578, | |
| "loss": 0.4453, | |
| "losses/dpo": 0.3751629889011383, | |
| "losses/sft": 0.20083920657634735, | |
| "losses/total": 0.3751629889011383, | |
| "ref_logps/chosen": -50.14271545410156, | |
| "ref_logps/rejected": -43.08906555175781, | |
| "rewards/accuracies": 0.78125, | |
| "rewards/chosen": -1.3514864444732666, | |
| "rewards/margins": 1.0182515382766724, | |
| "rewards/rejected": -2.3697378635406494, | |
| "step": 66 | |
| }, | |
| { | |
| "epoch": 0.92, | |
| "grad_norm": 32.512692665801914, | |
| "learning_rate": 1.0438542722708444e-08, | |
| "logps/chosen": -61.425235748291016, | |
| "logps/rejected": -62.99015426635742, | |
| "loss": 0.4034, | |
| "losses/dpo": 0.4745466709136963, | |
| "losses/sft": 0.15217916667461395, | |
| "losses/total": 0.4745466709136963, | |
| "ref_logps/chosen": -49.22258758544922, | |
| "ref_logps/rejected": -39.8096923828125, | |
| "rewards/accuracies": 0.84375, | |
| "rewards/chosen": -1.2202647924423218, | |
| "rewards/margins": 1.0977811813354492, | |
| "rewards/rejected": -2.3180460929870605, | |
| "step": 67 | |
| }, | |
| { | |
| "epoch": 0.93, | |
| "grad_norm": 28.142561520937964, | |
| "learning_rate": 7.2645456434869965e-09, | |
| "logps/chosen": -60.124473571777344, | |
| "logps/rejected": -64.52591705322266, | |
| "loss": 0.4073, | |
| "losses/dpo": 0.5824078917503357, | |
| "losses/sft": 0.15901198983192444, | |
| "losses/total": 0.5824078917503357, | |
| "ref_logps/chosen": -49.08730697631836, | |
| "ref_logps/rejected": -42.66169357299805, | |
| "rewards/accuracies": 0.8203125, | |
| "rewards/chosen": -1.1037168502807617, | |
| "rewards/margins": 1.0827052593231201, | |
| "rewards/rejected": -2.1864218711853027, | |
| "step": 68 | |
| }, | |
| { | |
| "epoch": 0.95, | |
| "grad_norm": 26.50196447879399, | |
| "learning_rate": 4.657468465146641e-09, | |
| "logps/chosen": -63.6911735534668, | |
| "logps/rejected": -68.59868621826172, | |
| "loss": 0.3973, | |
| "losses/dpo": 0.40227654576301575, | |
| "losses/sft": 0.18853811919689178, | |
| "losses/total": 0.40227654576301575, | |
| "ref_logps/chosen": -51.581443786621094, | |
| "ref_logps/rejected": -44.77821350097656, | |
| "rewards/accuracies": 0.8515625, | |
| "rewards/chosen": -1.210972785949707, | |
| "rewards/margins": 1.1710745096206665, | |
| "rewards/rejected": -2.382047176361084, | |
| "step": 69 | |
| }, | |
| { | |
| "epoch": 0.96, | |
| "grad_norm": 56.71695513519511, | |
| "learning_rate": 2.6234001372372193e-09, | |
| "logps/chosen": -63.47015380859375, | |
| "logps/rejected": -65.77841186523438, | |
| "loss": 0.4462, | |
| "losses/dpo": 0.6931471824645996, | |
| "losses/sft": 0.16308167576789856, | |
| "losses/total": 0.6931471824645996, | |
| "ref_logps/chosen": -50.47125244140625, | |
| "ref_logps/rejected": -43.315406799316406, | |
| "rewards/accuracies": 0.8203125, | |
| "rewards/chosen": -1.2998902797698975, | |
| "rewards/margins": 0.9464105367660522, | |
| "rewards/rejected": -2.24630069732666, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.97, | |
| "grad_norm": 23.674451477206517, | |
| "learning_rate": 1.167091320587843e-09, | |
| "logps/chosen": -62.78862762451172, | |
| "logps/rejected": -67.10327911376953, | |
| "loss": 0.4313, | |
| "losses/dpo": 0.12619969248771667, | |
| "losses/sft": 0.16695602238178253, | |
| "losses/total": 0.12619969248771667, | |
| "ref_logps/chosen": -50.83470153808594, | |
| "ref_logps/rejected": -45.14189147949219, | |
| "rewards/accuracies": 0.8046875, | |
| "rewards/chosen": -1.1953924894332886, | |
| "rewards/margins": 1.000746250152588, | |
| "rewards/rejected": -2.196138858795166, | |
| "step": 71 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "grad_norm": 26.52528545158649, | |
| "learning_rate": 2.9194329191833953e-10, | |
| "logps/chosen": -63.891353607177734, | |
| "logps/rejected": -66.84733581542969, | |
| "loss": 0.4241, | |
| "losses/dpo": 0.2748945951461792, | |
| "losses/sft": 0.1392793506383896, | |
| "losses/total": 0.2748945951461792, | |
| "ref_logps/chosen": -51.043392181396484, | |
| "ref_logps/rejected": -43.61123275756836, | |
| "rewards/accuracies": 0.7734375, | |
| "rewards/chosen": -1.2847963571548462, | |
| "rewards/margins": 1.0388145446777344, | |
| "rewards/rejected": -2.323610782623291, | |
| "step": 72 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "grad_norm": 32.933679564175364, | |
| "learning_rate": 0.0, | |
| "logps/chosen": -64.90662384033203, | |
| "logps/rejected": -65.81568145751953, | |
| "loss": 0.3989, | |
| "losses/dpo": 0.6386691927909851, | |
| "losses/sft": 0.17253142595291138, | |
| "losses/total": 0.6386691927909851, | |
| "ref_logps/chosen": -53.89427185058594, | |
| "ref_logps/rejected": -43.687034606933594, | |
| "rewards/accuracies": 0.8515625, | |
| "rewards/chosen": -1.1012351512908936, | |
| "rewards/margins": 1.1116297245025635, | |
| "rewards/rejected": -2.212864875793457, | |
| "step": 73 | |
| }, | |
| { | |
| "epoch": 1.0, | |
| "step": 73, | |
| "total_flos": 0.0, | |
| "train_loss": 0.5157321454727486, | |
| "train_runtime": 1200.6075, | |
| "train_samples_per_second": 7.848, | |
| "train_steps_per_second": 0.061 | |
| } | |
| ], | |
| "logging_steps": 1.0, | |
| "max_steps": 73, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 20000, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |