| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 2.973977695167286, | |
| "eval_steps": 500, | |
| "global_step": 300, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.05, | |
| "grad_norm": 2144.0, | |
| "learning_rate": 8.333333333333333e-07, | |
| "logits/chosen": -2.582127094268799, | |
| "logits/rejected": -2.6067237854003906, | |
| "logps/chosen": -1126.8817138671875, | |
| "logps/rejected": -1078.447509765625, | |
| "loss": 0.6576, | |
| "rewards/accuracies": 0.42500001192092896, | |
| "rewards/chosen": 0.08093704283237457, | |
| "rewards/margins": 0.09326125681400299, | |
| "rewards/rejected": -0.012324221432209015, | |
| "step": 5 | |
| }, | |
| { | |
| "epoch": 0.1, | |
| "grad_norm": 504.0, | |
| "learning_rate": 1.6666666666666667e-06, | |
| "logits/chosen": -2.340122699737549, | |
| "logits/rejected": -2.652778387069702, | |
| "logps/chosen": -1099.25048828125, | |
| "logps/rejected": -1232.9901123046875, | |
| "loss": 0.2009, | |
| "rewards/accuracies": 0.925000011920929, | |
| "rewards/chosen": 1.4397642612457275, | |
| "rewards/margins": 2.55800461769104, | |
| "rewards/rejected": -1.1182401180267334, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.15, | |
| "grad_norm": 36.0, | |
| "learning_rate": 2.5e-06, | |
| "logits/chosen": -2.513775110244751, | |
| "logits/rejected": -2.662311315536499, | |
| "logps/chosen": -990.4671020507812, | |
| "logps/rejected": -1043.4093017578125, | |
| "loss": 0.0709, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 2.9243946075439453, | |
| "rewards/margins": 6.777369499206543, | |
| "rewards/rejected": -3.8529744148254395, | |
| "step": 15 | |
| }, | |
| { | |
| "epoch": 0.2, | |
| "grad_norm": 3.703125, | |
| "learning_rate": 3.3333333333333333e-06, | |
| "logits/chosen": -2.4629788398742676, | |
| "logits/rejected": -2.802347183227539, | |
| "logps/chosen": -1008.41064453125, | |
| "logps/rejected": -1297.322265625, | |
| "loss": 0.0009, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 8.369072914123535, | |
| "rewards/margins": 15.539718627929688, | |
| "rewards/rejected": -7.170646667480469, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.25, | |
| "grad_norm": 0.000362396240234375, | |
| "learning_rate": 4.166666666666667e-06, | |
| "logits/chosen": -2.5417537689208984, | |
| "logits/rejected": -2.6880087852478027, | |
| "logps/chosen": -926.39892578125, | |
| "logps/rejected": -1187.432861328125, | |
| "loss": 0.0002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 10.67939281463623, | |
| "rewards/margins": 24.91078758239746, | |
| "rewards/rejected": -14.23139476776123, | |
| "step": 25 | |
| }, | |
| { | |
| "epoch": 0.3, | |
| "grad_norm": 0.048095703125, | |
| "learning_rate": 5e-06, | |
| "logits/chosen": -2.577080249786377, | |
| "logits/rejected": -2.733059883117676, | |
| "logps/chosen": -908.2595825195312, | |
| "logps/rejected": -1309.6556396484375, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 8.903298377990723, | |
| "rewards/margins": 34.511558532714844, | |
| "rewards/rejected": -25.608257293701172, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.35, | |
| "grad_norm": 4.0978193283081055e-07, | |
| "learning_rate": 4.995770395678171e-06, | |
| "logits/chosen": -2.651310920715332, | |
| "logits/rejected": -2.6355812549591064, | |
| "logps/chosen": -873.0777587890625, | |
| "logps/rejected": -1328.905517578125, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 11.599026679992676, | |
| "rewards/margins": 44.44927215576172, | |
| "rewards/rejected": -32.850250244140625, | |
| "step": 35 | |
| }, | |
| { | |
| "epoch": 0.4, | |
| "grad_norm": 0.055908203125, | |
| "learning_rate": 4.983095894354858e-06, | |
| "logits/chosen": -2.5453743934631348, | |
| "logits/rejected": -2.668313503265381, | |
| "logps/chosen": -1026.492919921875, | |
| "logps/rejected": -1623.9166259765625, | |
| "loss": 0.0061, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 12.174985885620117, | |
| "rewards/margins": 48.78560256958008, | |
| "rewards/rejected": -36.610618591308594, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.45, | |
| "grad_norm": 0.0002899169921875, | |
| "learning_rate": 4.962019382530521e-06, | |
| "logits/chosen": -2.688816785812378, | |
| "logits/rejected": -2.8246970176696777, | |
| "logps/chosen": -866.5418090820312, | |
| "logps/rejected": -1398.448974609375, | |
| "loss": 0.0001, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 15.687978744506836, | |
| "rewards/margins": 49.26004409790039, | |
| "rewards/rejected": -33.57206344604492, | |
| "step": 45 | |
| }, | |
| { | |
| "epoch": 0.5, | |
| "grad_norm": 3.448803909122944e-09, | |
| "learning_rate": 4.93261217644956e-06, | |
| "logits/chosen": -2.555048704147339, | |
| "logits/rejected": -2.771728515625, | |
| "logps/chosen": -826.4230346679688, | |
| "logps/rejected": -1372.0035400390625, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 14.595741271972656, | |
| "rewards/margins": 46.010047912597656, | |
| "rewards/rejected": -31.414306640625, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.55, | |
| "grad_norm": 14.5, | |
| "learning_rate": 4.894973780788722e-06, | |
| "logits/chosen": -2.570012331008911, | |
| "logits/rejected": -2.613524913787842, | |
| "logps/chosen": -867.3160400390625, | |
| "logps/rejected": -1359.3463134765625, | |
| "loss": 0.0006, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 16.941835403442383, | |
| "rewards/margins": 41.53192901611328, | |
| "rewards/rejected": -24.590087890625, | |
| "step": 55 | |
| }, | |
| { | |
| "epoch": 0.59, | |
| "grad_norm": 7.420778274536133e-06, | |
| "learning_rate": 4.849231551964771e-06, | |
| "logits/chosen": -2.6153066158294678, | |
| "logits/rejected": -2.857034206390381, | |
| "logps/chosen": -774.2218627929688, | |
| "logps/rejected": -1181.09619140625, | |
| "loss": 0.0001, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 21.74435043334961, | |
| "rewards/margins": 31.659595489501953, | |
| "rewards/rejected": -9.915243148803711, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.64, | |
| "grad_norm": 0.00081634521484375, | |
| "learning_rate": 4.7955402672006855e-06, | |
| "logits/chosen": -2.738781690597534, | |
| "logits/rejected": -2.8861005306243896, | |
| "logps/chosen": -704.155517578125, | |
| "logps/rejected": -1052.4534912109375, | |
| "loss": 0.0002, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 25.42382049560547, | |
| "rewards/margins": 30.588428497314453, | |
| "rewards/rejected": -5.164605617523193, | |
| "step": 65 | |
| }, | |
| { | |
| "epoch": 0.69, | |
| "grad_norm": 1.555308699607849e-07, | |
| "learning_rate": 4.734081600808531e-06, | |
| "logits/chosen": -2.744210720062256, | |
| "logits/rejected": -2.9681973457336426, | |
| "logps/chosen": -755.2449340820312, | |
| "logps/rejected": -1124.8590087890625, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 26.51047706604004, | |
| "rewards/margins": 34.19602966308594, | |
| "rewards/rejected": -7.685556888580322, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.74, | |
| "grad_norm": 0.0208740234375, | |
| "learning_rate": 4.665063509461098e-06, | |
| "logits/chosen": -2.6066527366638184, | |
| "logits/rejected": -2.763363838195801, | |
| "logps/chosen": -718.3406982421875, | |
| "logps/rejected": -1171.5050048828125, | |
| "loss": 0.0001, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 23.655757904052734, | |
| "rewards/margins": 39.56992721557617, | |
| "rewards/rejected": -15.914169311523438, | |
| "step": 75 | |
| }, | |
| { | |
| "epoch": 0.79, | |
| "grad_norm": 2.0605511963367462e-08, | |
| "learning_rate": 4.588719528532342e-06, | |
| "logits/chosen": -2.6591174602508545, | |
| "logits/rejected": -2.818765640258789, | |
| "logps/chosen": -713.7478637695312, | |
| "logps/rejected": -1214.9329833984375, | |
| "loss": 0.0178, | |
| "rewards/accuracies": 0.9750000238418579, | |
| "rewards/chosen": 25.415401458740234, | |
| "rewards/margins": 46.337589263916016, | |
| "rewards/rejected": -20.922182083129883, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.84, | |
| "grad_norm": 8.866190910339355e-07, | |
| "learning_rate": 4.50530798188761e-06, | |
| "logits/chosen": -2.592603921890259, | |
| "logits/rejected": -2.5666003227233887, | |
| "logps/chosen": -672.0972900390625, | |
| "logps/rejected": -1153.8974609375, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 21.85224151611328, | |
| "rewards/margins": 44.60143280029297, | |
| "rewards/rejected": -22.749195098876953, | |
| "step": 85 | |
| }, | |
| { | |
| "epoch": 0.89, | |
| "grad_norm": 0.0032501220703125, | |
| "learning_rate": 4.415111107797445e-06, | |
| "logits/chosen": -2.5358242988586426, | |
| "logits/rejected": -2.7607996463775635, | |
| "logps/chosen": -837.4728393554688, | |
| "logps/rejected": -1421.790771484375, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 21.30091667175293, | |
| "rewards/margins": 51.24127197265625, | |
| "rewards/rejected": -29.94034767150879, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.94, | |
| "grad_norm": 0.00067901611328125, | |
| "learning_rate": 4.318434103932622e-06, | |
| "logits/chosen": -2.5782151222229004, | |
| "logits/rejected": -2.708712339401245, | |
| "logps/chosen": -784.4146728515625, | |
| "logps/rejected": -1424.0408935546875, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 24.907550811767578, | |
| "rewards/margins": 59.107765197753906, | |
| "rewards/rejected": -34.20021438598633, | |
| "step": 95 | |
| }, | |
| { | |
| "epoch": 0.99, | |
| "grad_norm": 1.0652001947164536e-08, | |
| "learning_rate": 4.215604094671835e-06, | |
| "logits/chosen": -2.598839044570923, | |
| "logits/rejected": -2.6679890155792236, | |
| "logps/chosen": -733.5455322265625, | |
| "logps/rejected": -1301.297119140625, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 27.65069007873535, | |
| "rewards/margins": 55.6466178894043, | |
| "rewards/rejected": -27.995929718017578, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 1.04, | |
| "grad_norm": 3.073364496231079e-08, | |
| "learning_rate": 4.106969024216348e-06, | |
| "logits/chosen": -2.4979000091552734, | |
| "logits/rejected": -2.702669620513916, | |
| "logps/chosen": -800.4815673828125, | |
| "logps/rejected": -1545.734130859375, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 28.3299503326416, | |
| "rewards/margins": 66.7006607055664, | |
| "rewards/rejected": -38.37070846557617, | |
| "step": 105 | |
| }, | |
| { | |
| "epoch": 1.09, | |
| "grad_norm": 1.1980533599853516e-05, | |
| "learning_rate": 3.992896479256966e-06, | |
| "logits/chosen": -2.5696396827697754, | |
| "logits/rejected": -2.6518044471740723, | |
| "logps/chosen": -811.0010986328125, | |
| "logps/rejected": -1455.451904296875, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 24.791704177856445, | |
| "rewards/margins": 59.291259765625, | |
| "rewards/rejected": -34.49955749511719, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 1.14, | |
| "grad_norm": 1.178705133497715e-09, | |
| "learning_rate": 3.8737724451770155e-06, | |
| "logits/chosen": -2.480198383331299, | |
| "logits/rejected": -2.7152700424194336, | |
| "logps/chosen": -748.87255859375, | |
| "logps/rejected": -1521.2923583984375, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 24.639190673828125, | |
| "rewards/margins": 62.03853225708008, | |
| "rewards/rejected": -37.39933395385742, | |
| "step": 115 | |
| }, | |
| { | |
| "epoch": 1.19, | |
| "grad_norm": 1.7508864402770996e-07, | |
| "learning_rate": 3.7500000000000005e-06, | |
| "logits/chosen": -2.5954699516296387, | |
| "logits/rejected": -2.7862911224365234, | |
| "logps/chosen": -828.2017822265625, | |
| "logps/rejected": -1500.0533447265625, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 24.49906349182129, | |
| "rewards/margins": 57.625587463378906, | |
| "rewards/rejected": -33.12652587890625, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 1.24, | |
| "grad_norm": 2.9921531677246094e-05, | |
| "learning_rate": 3.621997950501156e-06, | |
| "logits/chosen": -2.5512304306030273, | |
| "logits/rejected": -2.5572822093963623, | |
| "logps/chosen": -721.7669067382812, | |
| "logps/rejected": -1303.0982666015625, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 24.202795028686523, | |
| "rewards/margins": 57.88828659057617, | |
| "rewards/rejected": -33.68549346923828, | |
| "step": 125 | |
| }, | |
| { | |
| "epoch": 1.29, | |
| "grad_norm": 1.0477378964424133e-09, | |
| "learning_rate": 3.4901994150978926e-06, | |
| "logits/chosen": -2.5931153297424316, | |
| "logits/rejected": -2.693915843963623, | |
| "logps/chosen": -727.4300537109375, | |
| "logps/rejected": -1358.4566650390625, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 24.488115310668945, | |
| "rewards/margins": 57.87506866455078, | |
| "rewards/rejected": -33.38694763183594, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 1.34, | |
| "grad_norm": 0.0002765655517578125, | |
| "learning_rate": 3.3550503583141726e-06, | |
| "logits/chosen": -2.6825664043426514, | |
| "logits/rejected": -2.786104679107666, | |
| "logps/chosen": -796.1810302734375, | |
| "logps/rejected": -1376.611328125, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 25.74446678161621, | |
| "rewards/margins": 58.13447189331055, | |
| "rewards/rejected": -32.39000701904297, | |
| "step": 135 | |
| }, | |
| { | |
| "epoch": 1.39, | |
| "grad_norm": 5.030632019042969e-05, | |
| "learning_rate": 3.217008081777726e-06, | |
| "logits/chosen": -2.5538275241851807, | |
| "logits/rejected": -2.7200071811676025, | |
| "logps/chosen": -782.6590576171875, | |
| "logps/rejected": -1443.415283203125, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 27.598276138305664, | |
| "rewards/margins": 58.656410217285156, | |
| "rewards/rejected": -31.05812644958496, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 1.44, | |
| "grad_norm": 3.6209821701049805e-06, | |
| "learning_rate": 3.0765396768561005e-06, | |
| "logits/chosen": -2.6117515563964844, | |
| "logits/rejected": -2.74174165725708, | |
| "logps/chosen": -670.0432739257812, | |
| "logps/rejected": -1294.08642578125, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 26.823715209960938, | |
| "rewards/margins": 60.58592987060547, | |
| "rewards/rejected": -33.76221466064453, | |
| "step": 145 | |
| }, | |
| { | |
| "epoch": 1.49, | |
| "grad_norm": 1.1932570487260818e-09, | |
| "learning_rate": 2.9341204441673267e-06, | |
| "logits/chosen": -2.6705422401428223, | |
| "logits/rejected": -2.7561261653900146, | |
| "logps/chosen": -830.1842041015625, | |
| "logps/rejected": -1481.5352783203125, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 24.070798873901367, | |
| "rewards/margins": 62.027305603027344, | |
| "rewards/rejected": -37.956504821777344, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 1.54, | |
| "grad_norm": 7.927417755126953e-06, | |
| "learning_rate": 2.7902322853130758e-06, | |
| "logits/chosen": -2.5545754432678223, | |
| "logits/rejected": -2.740570068359375, | |
| "logps/chosen": -853.0714111328125, | |
| "logps/rejected": -1508.432373046875, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 25.61324119567871, | |
| "rewards/margins": 58.5531120300293, | |
| "rewards/rejected": -32.93987274169922, | |
| "step": 155 | |
| }, | |
| { | |
| "epoch": 1.59, | |
| "grad_norm": 1.5178830414797062e-16, | |
| "learning_rate": 2.6453620722761897e-06, | |
| "logits/chosen": -2.5684103965759277, | |
| "logits/rejected": -2.6356844902038574, | |
| "logps/chosen": -695.0123291015625, | |
| "logps/rejected": -1333.6201171875, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 26.32758140563965, | |
| "rewards/margins": 56.49725341796875, | |
| "rewards/rejected": -30.169677734375, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 1.64, | |
| "grad_norm": 4.0512531995773315e-08, | |
| "learning_rate": 2.5e-06, | |
| "logits/chosen": -2.6028122901916504, | |
| "logits/rejected": -2.621495246887207, | |
| "logps/chosen": -689.859130859375, | |
| "logps/rejected": -1285.8802490234375, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 26.484960556030273, | |
| "rewards/margins": 60.581703186035156, | |
| "rewards/rejected": -34.09674835205078, | |
| "step": 165 | |
| }, | |
| { | |
| "epoch": 1.69, | |
| "grad_norm": 5.811452865600586e-07, | |
| "learning_rate": 2.3546379277238107e-06, | |
| "logits/chosen": -2.5608181953430176, | |
| "logits/rejected": -2.633148431777954, | |
| "logps/chosen": -689.5059204101562, | |
| "logps/rejected": -1259.192138671875, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 24.118501663208008, | |
| "rewards/margins": 56.985939025878906, | |
| "rewards/rejected": -32.86743927001953, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 1.73, | |
| "grad_norm": 2.637505531311035e-06, | |
| "learning_rate": 2.2097677146869242e-06, | |
| "logits/chosen": -2.5463547706604004, | |
| "logits/rejected": -2.6621968746185303, | |
| "logps/chosen": -845.2557373046875, | |
| "logps/rejected": -1509.9571533203125, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 20.753326416015625, | |
| "rewards/margins": 60.85497283935547, | |
| "rewards/rejected": -40.101646423339844, | |
| "step": 175 | |
| }, | |
| { | |
| "epoch": 1.78, | |
| "grad_norm": 2.8033131371785203e-15, | |
| "learning_rate": 2.0658795558326745e-06, | |
| "logits/chosen": -2.5142176151275635, | |
| "logits/rejected": -2.800828456878662, | |
| "logps/chosen": -768.0780029296875, | |
| "logps/rejected": -1504.419677734375, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 26.789424896240234, | |
| "rewards/margins": 65.28126525878906, | |
| "rewards/rejected": -38.49182891845703, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 1.83, | |
| "grad_norm": 7.450580596923828e-06, | |
| "learning_rate": 1.9234603231439e-06, | |
| "logits/chosen": -2.5999841690063477, | |
| "logits/rejected": -2.785568952560425, | |
| "logps/chosen": -742.1795654296875, | |
| "logps/rejected": -1414.2978515625, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 26.651920318603516, | |
| "rewards/margins": 61.58074951171875, | |
| "rewards/rejected": -34.92882537841797, | |
| "step": 185 | |
| }, | |
| { | |
| "epoch": 1.88, | |
| "grad_norm": 3.841705620288849e-08, | |
| "learning_rate": 1.7829919182222752e-06, | |
| "logits/chosen": -2.735452175140381, | |
| "logits/rejected": -2.7289414405822754, | |
| "logps/chosen": -814.5637817382812, | |
| "logps/rejected": -1430.7174072265625, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 28.13421630859375, | |
| "rewards/margins": 61.936744689941406, | |
| "rewards/rejected": -33.802528381347656, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 1.93, | |
| "grad_norm": 6.892264536872972e-13, | |
| "learning_rate": 1.6449496416858285e-06, | |
| "logits/chosen": -2.515446186065674, | |
| "logits/rejected": -2.742518663406372, | |
| "logps/chosen": -745.94970703125, | |
| "logps/rejected": -1403.6351318359375, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 26.278427124023438, | |
| "rewards/margins": 64.49276733398438, | |
| "rewards/rejected": -38.21433639526367, | |
| "step": 195 | |
| }, | |
| { | |
| "epoch": 1.98, | |
| "grad_norm": 1.1757947504520416e-08, | |
| "learning_rate": 1.509800584902108e-06, | |
| "logits/chosen": -2.478830575942993, | |
| "logits/rejected": -2.7162294387817383, | |
| "logps/chosen": -780.1927490234375, | |
| "logps/rejected": -1506.2659912109375, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 26.495025634765625, | |
| "rewards/margins": 63.679115295410156, | |
| "rewards/rejected": -37.184085845947266, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 2.03, | |
| "grad_norm": 1.519918441772461e-05, | |
| "learning_rate": 1.3780020494988447e-06, | |
| "logits/chosen": -2.5534565448760986, | |
| "logits/rejected": -2.703151226043701, | |
| "logps/chosen": -684.3609619140625, | |
| "logps/rejected": -1303.3043212890625, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 23.173385620117188, | |
| "rewards/margins": 54.865631103515625, | |
| "rewards/rejected": -31.692245483398438, | |
| "step": 205 | |
| }, | |
| { | |
| "epoch": 2.08, | |
| "grad_norm": 0.0001277923583984375, | |
| "learning_rate": 1.2500000000000007e-06, | |
| "logits/chosen": -2.566140651702881, | |
| "logits/rejected": -2.703296661376953, | |
| "logps/chosen": -764.0347290039062, | |
| "logps/rejected": -1441.303955078125, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 25.557802200317383, | |
| "rewards/margins": 61.8077392578125, | |
| "rewards/rejected": -36.249942779541016, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 2.13, | |
| "grad_norm": 3.110617399215698e-07, | |
| "learning_rate": 1.1262275548229852e-06, | |
| "logits/chosen": -2.577171564102173, | |
| "logits/rejected": -2.6875264644622803, | |
| "logps/chosen": -733.8834228515625, | |
| "logps/rejected": -1341.32763671875, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 23.420785903930664, | |
| "rewards/margins": 59.020118713378906, | |
| "rewards/rejected": -35.59933090209961, | |
| "step": 215 | |
| }, | |
| { | |
| "epoch": 2.18, | |
| "grad_norm": 1.8917489796876907e-09, | |
| "learning_rate": 1.0071035207430352e-06, | |
| "logits/chosen": -2.5544378757476807, | |
| "logits/rejected": -2.641348361968994, | |
| "logps/chosen": -771.989013671875, | |
| "logps/rejected": -1348.832275390625, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 24.312747955322266, | |
| "rewards/margins": 58.93672561645508, | |
| "rewards/rejected": -34.62397003173828, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 2.23, | |
| "grad_norm": 8.221832104027271e-10, | |
| "learning_rate": 8.930309757836517e-07, | |
| "logits/chosen": -2.6528477668762207, | |
| "logits/rejected": -2.757702350616455, | |
| "logps/chosen": -788.48828125, | |
| "logps/rejected": -1497.994140625, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 24.537336349487305, | |
| "rewards/margins": 68.33438110351562, | |
| "rewards/rejected": -43.79704284667969, | |
| "step": 225 | |
| }, | |
| { | |
| "epoch": 2.28, | |
| "grad_norm": 0.000446319580078125, | |
| "learning_rate": 7.843959053281663e-07, | |
| "logits/chosen": -2.631401777267456, | |
| "logits/rejected": -2.7087044715881348, | |
| "logps/chosen": -811.6467895507812, | |
| "logps/rejected": -1447.6087646484375, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 25.546356201171875, | |
| "rewards/margins": 62.1228141784668, | |
| "rewards/rejected": -36.576454162597656, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 2.33, | |
| "grad_norm": 0.001556396484375, | |
| "learning_rate": 6.815658960673782e-07, | |
| "logits/chosen": -2.496450901031494, | |
| "logits/rejected": -2.640903949737549, | |
| "logps/chosen": -756.075439453125, | |
| "logps/rejected": -1518.998046875, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 22.391010284423828, | |
| "rewards/margins": 63.806968688964844, | |
| "rewards/rejected": -41.41596603393555, | |
| "step": 235 | |
| }, | |
| { | |
| "epoch": 2.38, | |
| "grad_norm": 3.361701965332031e-05, | |
| "learning_rate": 5.848888922025553e-07, | |
| "logits/chosen": -2.5900871753692627, | |
| "logits/rejected": -2.6478281021118164, | |
| "logps/chosen": -654.4774780273438, | |
| "logps/rejected": -1188.7408447265625, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 25.036846160888672, | |
| "rewards/margins": 53.53025436401367, | |
| "rewards/rejected": -28.493408203125, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 2.43, | |
| "grad_norm": 2.1191226551309228e-10, | |
| "learning_rate": 4.946920181123904e-07, | |
| "logits/chosen": -2.509355068206787, | |
| "logits/rejected": -2.687610149383545, | |
| "logps/chosen": -787.9483642578125, | |
| "logps/rejected": -1455.489013671875, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 23.417675018310547, | |
| "rewards/margins": 57.781227111816406, | |
| "rewards/rejected": -34.363548278808594, | |
| "step": 245 | |
| }, | |
| { | |
| "epoch": 2.48, | |
| "grad_norm": 1.210719347000122e-08, | |
| "learning_rate": 4.1128047146765936e-07, | |
| "logits/chosen": -2.5654799938201904, | |
| "logits/rejected": -2.7495007514953613, | |
| "logps/chosen": -880.7611083984375, | |
| "logps/rejected": -1492.9364013671875, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 26.42355728149414, | |
| "rewards/margins": 58.61255645751953, | |
| "rewards/rejected": -32.188995361328125, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 2.53, | |
| "grad_norm": 3.552713678800501e-12, | |
| "learning_rate": 3.3493649053890325e-07, | |
| "logits/chosen": -2.5684847831726074, | |
| "logits/rejected": -2.7665956020355225, | |
| "logps/chosen": -717.5662231445312, | |
| "logps/rejected": -1405.523681640625, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 25.795669555664062, | |
| "rewards/margins": 55.92620849609375, | |
| "rewards/rejected": -30.130529403686523, | |
| "step": 255 | |
| }, | |
| { | |
| "epoch": 2.58, | |
| "grad_norm": 4.1470733097570545e-18, | |
| "learning_rate": 2.6591839919146963e-07, | |
| "logits/chosen": -2.5511410236358643, | |
| "logits/rejected": -2.642638921737671, | |
| "logps/chosen": -770.4322509765625, | |
| "logps/rejected": -1513.5374755859375, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 26.562414169311523, | |
| "rewards/margins": 67.80595397949219, | |
| "rewards/rejected": -41.24353790283203, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 2.63, | |
| "grad_norm": 2.775341272354126e-07, | |
| "learning_rate": 2.044597327993153e-07, | |
| "logits/chosen": -2.5555355548858643, | |
| "logits/rejected": -2.715888738632202, | |
| "logps/chosen": -807.5404052734375, | |
| "logps/rejected": -1584.75927734375, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 25.03497886657715, | |
| "rewards/margins": 64.63668823242188, | |
| "rewards/rejected": -39.601707458496094, | |
| "step": 265 | |
| }, | |
| { | |
| "epoch": 2.68, | |
| "grad_norm": 4.007461029686965e-12, | |
| "learning_rate": 1.507684480352292e-07, | |
| "logits/chosen": -2.597036123275757, | |
| "logits/rejected": -2.643857479095459, | |
| "logps/chosen": -831.2688598632812, | |
| "logps/rejected": -1632.8023681640625, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 24.538908004760742, | |
| "rewards/margins": 74.41605377197266, | |
| "rewards/rejected": -49.87714385986328, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 2.73, | |
| "grad_norm": 4.172325134277344e-05, | |
| "learning_rate": 1.0502621921127776e-07, | |
| "logits/chosen": -2.5798580646514893, | |
| "logits/rejected": -2.7736194133758545, | |
| "logps/chosen": -767.0318603515625, | |
| "logps/rejected": -1508.767822265625, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 25.648067474365234, | |
| "rewards/margins": 66.6827163696289, | |
| "rewards/rejected": -41.034645080566406, | |
| "step": 275 | |
| }, | |
| { | |
| "epoch": 2.78, | |
| "grad_norm": 1.9818544387817383e-06, | |
| "learning_rate": 6.738782355044048e-08, | |
| "logits/chosen": -2.5585246086120605, | |
| "logits/rejected": -2.666764736175537, | |
| "logps/chosen": -772.8341064453125, | |
| "logps/rejected": -1421.703857421875, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 27.29400634765625, | |
| "rewards/margins": 63.170677185058594, | |
| "rewards/rejected": -35.87666702270508, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 2.83, | |
| "grad_norm": 1.2631062418222427e-08, | |
| "learning_rate": 3.798061746947995e-08, | |
| "logits/chosen": -2.512103319168091, | |
| "logits/rejected": -2.690737009048462, | |
| "logps/chosen": -810.3479614257812, | |
| "logps/rejected": -1493.773193359375, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 22.784202575683594, | |
| "rewards/margins": 61.64385223388672, | |
| "rewards/rejected": -38.859649658203125, | |
| "step": 285 | |
| }, | |
| { | |
| "epoch": 2.87, | |
| "grad_norm": 8.009374141693115e-08, | |
| "learning_rate": 1.6904105645142443e-08, | |
| "logits/chosen": -2.618516445159912, | |
| "logits/rejected": -2.7000651359558105, | |
| "logps/chosen": -769.9617309570312, | |
| "logps/rejected": -1351.812255859375, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 25.189565658569336, | |
| "rewards/margins": 57.4492073059082, | |
| "rewards/rejected": -32.259639739990234, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 2.92, | |
| "grad_norm": 9.324138683375338e-17, | |
| "learning_rate": 4.229604321829561e-09, | |
| "logits/chosen": -2.5340938568115234, | |
| "logits/rejected": -2.880659818649292, | |
| "logps/chosen": -801.6094970703125, | |
| "logps/rejected": -1518.06201171875, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 26.99312400817871, | |
| "rewards/margins": 64.31195068359375, | |
| "rewards/rejected": -37.3188362121582, | |
| "step": 295 | |
| }, | |
| { | |
| "epoch": 2.97, | |
| "grad_norm": 3.241002559661865e-07, | |
| "learning_rate": 0.0, | |
| "logits/chosen": -2.616868019104004, | |
| "logits/rejected": -2.702866792678833, | |
| "logps/chosen": -725.1414184570312, | |
| "logps/rejected": -1320.96435546875, | |
| "loss": 0.0, | |
| "rewards/accuracies": 1.0, | |
| "rewards/chosen": 25.64286231994629, | |
| "rewards/margins": 60.58977508544922, | |
| "rewards/rejected": -34.94690704345703, | |
| "step": 300 | |
| } | |
| ], | |
| "logging_steps": 5, | |
| "max_steps": 300, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "total_flos": 0.0, | |
| "train_batch_size": 1, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |