| { | |
| "best_metric": 0.8434417247772217, | |
| "best_model_checkpoint": "saves/Mistral-7B-Instruct-v0.3/lora/orpo-salt/checkpoint-1500", | |
| "epoch": 2.9969690846635686, | |
| "eval_steps": 500, | |
| "global_step": 1854, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.01616488179430188, | |
| "grad_norm": 8.316998481750488, | |
| "learning_rate": 4.999648198770648e-06, | |
| "logits/chosen": -2.9437620639801025, | |
| "logits/rejected": -2.991391658782959, | |
| "logps/chosen": -1.0850014686584473, | |
| "logps/rejected": -1.700299620628357, | |
| "loss": 1.1424, | |
| "odds_ratio_loss": 0.5736632347106934, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.10850014537572861, | |
| "rewards/margins": 0.06152981519699097, | |
| "rewards/rejected": -0.17002995312213898, | |
| "sft_loss": 1.0850014686584473, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.03232976358860376, | |
| "grad_norm": 11.040986061096191, | |
| "learning_rate": 4.998578646361359e-06, | |
| "logits/chosen": -2.942950963973999, | |
| "logits/rejected": -2.972404956817627, | |
| "logps/chosen": -1.057308316230774, | |
| "logps/rejected": -1.3991749286651611, | |
| "loss": 1.1175, | |
| "odds_ratio_loss": 0.6021074056625366, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -0.10573084652423859, | |
| "rewards/margins": 0.03418666869401932, | |
| "rewards/rejected": -0.1399175077676773, | |
| "sft_loss": 1.057308316230774, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.04849464538290564, | |
| "grad_norm": 6.759947776794434, | |
| "learning_rate": 4.996791614004449e-06, | |
| "logits/chosen": -2.967661142349243, | |
| "logits/rejected": -2.988191604614258, | |
| "logps/chosen": -0.9923480749130249, | |
| "logps/rejected": -1.4220160245895386, | |
| "loss": 1.0537, | |
| "odds_ratio_loss": 0.6132165789604187, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.09923480451107025, | |
| "rewards/margins": 0.042966801673173904, | |
| "rewards/rejected": -0.14220160245895386, | |
| "sft_loss": 0.9923480749130249, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.06465952717720752, | |
| "grad_norm": 7.137161731719971, | |
| "learning_rate": 4.994287614855618e-06, | |
| "logits/chosen": -2.920475721359253, | |
| "logits/rejected": -2.9866600036621094, | |
| "logps/chosen": -1.0413509607315063, | |
| "logps/rejected": -1.3548152446746826, | |
| "loss": 1.1081, | |
| "odds_ratio_loss": 0.6675835847854614, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.10413509607315063, | |
| "rewards/margins": 0.03134642913937569, | |
| "rewards/rejected": -0.13548150658607483, | |
| "sft_loss": 1.0413509607315063, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.0808244089715094, | |
| "grad_norm": 5.560673236846924, | |
| "learning_rate": 4.991067367951343e-06, | |
| "logits/chosen": -3.0365490913391113, | |
| "logits/rejected": -3.02650785446167, | |
| "logps/chosen": -1.0141496658325195, | |
| "logps/rejected": -1.311621904373169, | |
| "loss": 1.0767, | |
| "odds_ratio_loss": 0.6250823140144348, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.10141497850418091, | |
| "rewards/margins": 0.02974722720682621, | |
| "rewards/rejected": -0.13116219639778137, | |
| "sft_loss": 1.0141496658325195, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.09698929076581128, | |
| "grad_norm": 2.7108840942382812, | |
| "learning_rate": 4.987131798002389e-06, | |
| "logits/chosen": -2.9634835720062256, | |
| "logits/rejected": -2.984647274017334, | |
| "logps/chosen": -0.9159129858016968, | |
| "logps/rejected": -1.1770719289779663, | |
| "loss": 0.9847, | |
| "odds_ratio_loss": 0.6881905198097229, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.09159130603075027, | |
| "rewards/margins": 0.026115888729691505, | |
| "rewards/rejected": -0.11770719289779663, | |
| "sft_loss": 0.9159129858016968, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.11315417256011315, | |
| "grad_norm": 10.522608757019043, | |
| "learning_rate": 4.982482035128285e-06, | |
| "logits/chosen": -2.909318208694458, | |
| "logits/rejected": -2.938032388687134, | |
| "logps/chosen": -0.9783811569213867, | |
| "logps/rejected": -1.2935713529586792, | |
| "loss": 1.0458, | |
| "odds_ratio_loss": 0.6742582321166992, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.09783812612295151, | |
| "rewards/margins": 0.03151901811361313, | |
| "rewards/rejected": -0.12935714423656464, | |
| "sft_loss": 0.9783811569213867, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.12931905435441504, | |
| "grad_norm": 5.515076160430908, | |
| "learning_rate": 4.9771194145328e-06, | |
| "logits/chosen": -2.9371273517608643, | |
| "logits/rejected": -2.9489827156066895, | |
| "logps/chosen": -0.8305400013923645, | |
| "logps/rejected": -1.074385643005371, | |
| "loss": 0.8918, | |
| "odds_ratio_loss": 0.612562358379364, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.08305400609970093, | |
| "rewards/margins": 0.024384554475545883, | |
| "rewards/rejected": -0.10743856430053711, | |
| "sft_loss": 0.8305400013923645, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.1454839361487169, | |
| "grad_norm": 4.674871444702148, | |
| "learning_rate": 4.971045476120532e-06, | |
| "logits/chosen": -2.9331607818603516, | |
| "logits/rejected": -2.950925350189209, | |
| "logps/chosen": -0.828619122505188, | |
| "logps/rejected": -1.0910576581954956, | |
| "loss": 0.892, | |
| "odds_ratio_loss": 0.6336237192153931, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.08286191523075104, | |
| "rewards/margins": 0.02624385617673397, | |
| "rewards/rejected": -0.10910576581954956, | |
| "sft_loss": 0.828619122505188, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.1616488179430188, | |
| "grad_norm": 1.4944851398468018, | |
| "learning_rate": 4.964261964054713e-06, | |
| "logits/chosen": -2.902466297149658, | |
| "logits/rejected": -2.9229366779327393, | |
| "logps/chosen": -0.8629521131515503, | |
| "logps/rejected": -1.1251227855682373, | |
| "loss": 0.9278, | |
| "odds_ratio_loss": 0.6480029821395874, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.0862952247262001, | |
| "rewards/margins": 0.02621707320213318, | |
| "rewards/rejected": -0.11251229047775269, | |
| "sft_loss": 0.8629521131515503, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.17781369973732067, | |
| "grad_norm": 2.9030275344848633, | |
| "learning_rate": 4.956770826256372e-06, | |
| "logits/chosen": -2.957075595855713, | |
| "logits/rejected": -2.9636242389678955, | |
| "logps/chosen": -0.8815444707870483, | |
| "logps/rejected": -1.0787910223007202, | |
| "loss": 0.9485, | |
| "odds_ratio_loss": 0.6699932813644409, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.08815445005893707, | |
| "rewards/margins": 0.019724659621715546, | |
| "rewards/rejected": -0.10787911713123322, | |
| "sft_loss": 0.8815444707870483, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.19397858153162256, | |
| "grad_norm": 2.7190232276916504, | |
| "learning_rate": 4.94857421384497e-06, | |
| "logits/chosen": -2.9349093437194824, | |
| "logits/rejected": -2.958707332611084, | |
| "logps/chosen": -0.8882864117622375, | |
| "logps/rejected": -1.1156352758407593, | |
| "loss": 0.9568, | |
| "odds_ratio_loss": 0.6850704550743103, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.08882863819599152, | |
| "rewards/margins": 0.022734878584742546, | |
| "rewards/rejected": -0.11156351864337921, | |
| "sft_loss": 0.8882864117622375, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.21014346332592443, | |
| "grad_norm": 2.794067859649658, | |
| "learning_rate": 4.939674480520701e-06, | |
| "logits/chosen": -2.9200732707977295, | |
| "logits/rejected": -2.971010446548462, | |
| "logps/chosen": -0.9047578573226929, | |
| "logps/rejected": -1.1047561168670654, | |
| "loss": 0.971, | |
| "odds_ratio_loss": 0.6627197861671448, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.09047579020261765, | |
| "rewards/margins": 0.019999820739030838, | |
| "rewards/rejected": -0.11047561466693878, | |
| "sft_loss": 0.9047578573226929, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.2263083451202263, | |
| "grad_norm": 2.3947913646698, | |
| "learning_rate": 4.930074181888613e-06, | |
| "logits/chosen": -2.9679551124572754, | |
| "logits/rejected": -3.003051280975342, | |
| "logps/chosen": -0.8538404703140259, | |
| "logps/rejected": -1.0635920763015747, | |
| "loss": 0.9144, | |
| "odds_ratio_loss": 0.6060706377029419, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.0853840559720993, | |
| "rewards/margins": 0.020975153893232346, | |
| "rewards/rejected": -0.10635919868946075, | |
| "sft_loss": 0.8538404703140259, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.2424732269145282, | |
| "grad_norm": 1.2783300876617432, | |
| "learning_rate": 4.91977607472475e-06, | |
| "logits/chosen": -2.9910409450531006, | |
| "logits/rejected": -3.0015668869018555, | |
| "logps/chosen": -0.8571484684944153, | |
| "logps/rejected": -1.0444114208221436, | |
| "loss": 0.9202, | |
| "odds_ratio_loss": 0.6310030221939087, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.08571484684944153, | |
| "rewards/margins": 0.018726304173469543, | |
| "rewards/rejected": -0.10444115102291107, | |
| "sft_loss": 0.8571484684944153, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.2586381087088301, | |
| "grad_norm": 1.8696929216384888, | |
| "learning_rate": 4.908783116184534e-06, | |
| "logits/chosen": -2.924990177154541, | |
| "logits/rejected": -2.9346060752868652, | |
| "logps/chosen": -0.8140287399291992, | |
| "logps/rejected": -1.0593068599700928, | |
| "loss": 0.8729, | |
| "odds_ratio_loss": 0.5890231132507324, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.08140286803245544, | |
| "rewards/margins": 0.024527812376618385, | |
| "rewards/rejected": -0.10593068599700928, | |
| "sft_loss": 0.8140287399291992, | |
| "step": 160 | |
| }, | |
| { | |
| "epoch": 0.27480299050313195, | |
| "grad_norm": 1.7881205081939697, | |
| "learning_rate": 4.897098462953598e-06, | |
| "logits/chosen": -3.010953426361084, | |
| "logits/rejected": -3.0108227729797363, | |
| "logps/chosen": -0.8341928720474243, | |
| "logps/rejected": -1.1377969980239868, | |
| "loss": 0.8988, | |
| "odds_ratio_loss": 0.6464797258377075, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.08341928571462631, | |
| "rewards/margins": 0.030360404402017593, | |
| "rewards/rejected": -0.1137797012925148, | |
| "sft_loss": 0.8341928720474243, | |
| "step": 170 | |
| }, | |
| { | |
| "epoch": 0.2909678722974338, | |
| "grad_norm": 1.495703935623169, | |
| "learning_rate": 4.884725470341331e-06, | |
| "logits/chosen": -2.979276180267334, | |
| "logits/rejected": -3.003962755203247, | |
| "logps/chosen": -0.8357053995132446, | |
| "logps/rejected": -1.0890777111053467, | |
| "loss": 0.894, | |
| "odds_ratio_loss": 0.5833606123924255, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.08357055485248566, | |
| "rewards/margins": 0.025337230414152145, | |
| "rewards/rejected": -0.1089077740907669, | |
| "sft_loss": 0.8357053995132446, | |
| "step": 180 | |
| }, | |
| { | |
| "epoch": 0.3071327540917357, | |
| "grad_norm": 12.648097038269043, | |
| "learning_rate": 4.871667691317377e-06, | |
| "logits/chosen": -3.0223159790039062, | |
| "logits/rejected": -3.021965742111206, | |
| "logps/chosen": -0.9438085556030273, | |
| "logps/rejected": -0.9988954663276672, | |
| "loss": 1.0229, | |
| "odds_ratio_loss": 0.7914139628410339, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.09438085556030273, | |
| "rewards/margins": 0.005508692469447851, | |
| "rewards/rejected": -0.09988953918218613, | |
| "sft_loss": 0.9438085556030273, | |
| "step": 190 | |
| }, | |
| { | |
| "epoch": 0.3232976358860376, | |
| "grad_norm": 10.996000289916992, | |
| "learning_rate": 4.857928875491392e-06, | |
| "logits/chosen": -2.9932854175567627, | |
| "logits/rejected": -2.9929189682006836, | |
| "logps/chosen": -0.7825512886047363, | |
| "logps/rejected": -0.9704290628433228, | |
| "loss": 0.8476, | |
| "odds_ratio_loss": 0.65040123462677, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.07825513184070587, | |
| "rewards/margins": 0.018787771463394165, | |
| "rewards/rejected": -0.09704291075468063, | |
| "sft_loss": 0.7825512886047363, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.33946251768033947, | |
| "grad_norm": 2.8774170875549316, | |
| "learning_rate": 4.843512968036314e-06, | |
| "logits/chosen": -2.942992925643921, | |
| "logits/rejected": -2.9653749465942383, | |
| "logps/chosen": -0.7782715559005737, | |
| "logps/rejected": -0.9667309522628784, | |
| "loss": 0.8401, | |
| "odds_ratio_loss": 0.6183562874794006, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.07782715559005737, | |
| "rewards/margins": 0.018845947459340096, | |
| "rewards/rejected": -0.09667309373617172, | |
| "sft_loss": 0.7782715559005737, | |
| "step": 210 | |
| }, | |
| { | |
| "epoch": 0.35562739947464134, | |
| "grad_norm": 1.0692508220672607, | |
| "learning_rate": 4.828424108555486e-06, | |
| "logits/chosen": -3.0436058044433594, | |
| "logits/rejected": -3.0385215282440186, | |
| "logps/chosen": -0.9878608584403992, | |
| "logps/rejected": -1.2349365949630737, | |
| "loss": 1.0531, | |
| "odds_ratio_loss": 0.6525717377662659, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.09878608584403992, | |
| "rewards/margins": 0.024707583710551262, | |
| "rewards/rejected": -0.12349365651607513, | |
| "sft_loss": 0.9878608584403992, | |
| "step": 220 | |
| }, | |
| { | |
| "epoch": 0.3717922812689432, | |
| "grad_norm": 0.9835062026977539, | |
| "learning_rate": 4.812666629893957e-06, | |
| "logits/chosen": -3.0219979286193848, | |
| "logits/rejected": -3.0635628700256348, | |
| "logps/chosen": -0.8264273405075073, | |
| "logps/rejected": -0.9492254257202148, | |
| "loss": 0.8963, | |
| "odds_ratio_loss": 0.6982277631759644, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.08264274150133133, | |
| "rewards/margins": 0.012279799208045006, | |
| "rewards/rejected": -0.09492253512144089, | |
| "sft_loss": 0.8264273405075073, | |
| "step": 230 | |
| }, | |
| { | |
| "epoch": 0.3879571630632451, | |
| "grad_norm": 2.8942973613739014, | |
| "learning_rate": 4.796245056894273e-06, | |
| "logits/chosen": -2.9647586345672607, | |
| "logits/rejected": -3.0110132694244385, | |
| "logps/chosen": -0.8473427891731262, | |
| "logps/rejected": -1.0194810628890991, | |
| "loss": 0.9194, | |
| "odds_ratio_loss": 0.7206528782844543, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.08473427593708038, | |
| "rewards/margins": 0.01721382327377796, | |
| "rewards/rejected": -0.10194810479879379, | |
| "sft_loss": 0.8473427891731262, | |
| "step": 240 | |
| }, | |
| { | |
| "epoch": 0.404122044857547, | |
| "grad_norm": 1.4256747961044312, | |
| "learning_rate": 4.779164105097148e-06, | |
| "logits/chosen": -3.019832134246826, | |
| "logits/rejected": -3.0398221015930176, | |
| "logps/chosen": -0.7970795631408691, | |
| "logps/rejected": -1.0736204385757446, | |
| "loss": 0.859, | |
| "odds_ratio_loss": 0.6194061040878296, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.07970795035362244, | |
| "rewards/margins": 0.02765408717095852, | |
| "rewards/rejected": -0.1073620468378067, | |
| "sft_loss": 0.7970795631408691, | |
| "step": 250 | |
| }, | |
| { | |
| "epoch": 0.42028692665184886, | |
| "grad_norm": 2.344681739807129, | |
| "learning_rate": 4.761428679387373e-06, | |
| "logits/chosen": -3.003972291946411, | |
| "logits/rejected": -3.0536952018737793, | |
| "logps/chosen": -0.8031284213066101, | |
| "logps/rejected": -0.9748827815055847, | |
| "loss": 0.8698, | |
| "odds_ratio_loss": 0.666685163974762, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.0803128331899643, | |
| "rewards/margins": 0.017175443470478058, | |
| "rewards/rejected": -0.09748829156160355, | |
| "sft_loss": 0.8031284213066101, | |
| "step": 260 | |
| }, | |
| { | |
| "epoch": 0.4364518084461507, | |
| "grad_norm": 3.5396459102630615, | |
| "learning_rate": 4.7430438725853515e-06, | |
| "logits/chosen": -2.9764037132263184, | |
| "logits/rejected": -3.0066990852355957, | |
| "logps/chosen": -0.8188526034355164, | |
| "logps/rejected": -1.1920969486236572, | |
| "loss": 0.8795, | |
| "odds_ratio_loss": 0.6065649390220642, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.08188526332378387, | |
| "rewards/margins": 0.03732443228363991, | |
| "rewards/rejected": -0.11920969188213348, | |
| "sft_loss": 0.8188526034355164, | |
| "step": 270 | |
| }, | |
| { | |
| "epoch": 0.4526166902404526, | |
| "grad_norm": 2.1748006343841553, | |
| "learning_rate": 4.724014963984669e-06, | |
| "logits/chosen": -3.0459182262420654, | |
| "logits/rejected": -3.0646400451660156, | |
| "logps/chosen": -0.8359659910202026, | |
| "logps/rejected": -1.090301513671875, | |
| "loss": 0.9006, | |
| "odds_ratio_loss": 0.6466442942619324, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.0835966020822525, | |
| "rewards/margins": 0.02543354593217373, | |
| "rewards/rejected": -0.10903014987707138, | |
| "sft_loss": 0.8359659910202026, | |
| "step": 280 | |
| }, | |
| { | |
| "epoch": 0.4687815720347545, | |
| "grad_norm": 4.898100852966309, | |
| "learning_rate": 4.704347417836116e-06, | |
| "logits/chosen": -3.0006985664367676, | |
| "logits/rejected": -3.0678975582122803, | |
| "logps/chosen": -0.7437621355056763, | |
| "logps/rejected": -0.9980353116989136, | |
| "loss": 0.8091, | |
| "odds_ratio_loss": 0.6530498266220093, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.07437621057033539, | |
| "rewards/margins": 0.025427332147955894, | |
| "rewards/rejected": -0.09980354458093643, | |
| "sft_loss": 0.7437621355056763, | |
| "step": 290 | |
| }, | |
| { | |
| "epoch": 0.4849464538290564, | |
| "grad_norm": 1.6706643104553223, | |
| "learning_rate": 4.684046881778603e-06, | |
| "logits/chosen": -3.0016372203826904, | |
| "logits/rejected": -3.0139174461364746, | |
| "logps/chosen": -0.7875638008117676, | |
| "logps/rejected": -0.9269906282424927, | |
| "loss": 0.8522, | |
| "odds_ratio_loss": 0.6463567614555359, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.07875639200210571, | |
| "rewards/margins": 0.013942673802375793, | |
| "rewards/rejected": -0.0926990658044815, | |
| "sft_loss": 0.7875638008117676, | |
| "step": 300 | |
| }, | |
| { | |
| "epoch": 0.5011113356233583, | |
| "grad_norm": 2.1470396518707275, | |
| "learning_rate": 4.663119185217409e-06, | |
| "logits/chosen": -3.0117690563201904, | |
| "logits/rejected": -3.0633795261383057, | |
| "logps/chosen": -0.779082715511322, | |
| "logps/rejected": -1.0186303853988647, | |
| "loss": 0.8398, | |
| "odds_ratio_loss": 0.6070552468299866, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.07790827751159668, | |
| "rewards/margins": 0.023954764008522034, | |
| "rewards/rejected": -0.10186304897069931, | |
| "sft_loss": 0.779082715511322, | |
| "step": 310 | |
| }, | |
| { | |
| "epoch": 0.5172762174176602, | |
| "grad_norm": 1.40939199924469, | |
| "learning_rate": 4.641570337650232e-06, | |
| "logits/chosen": -3.0548150539398193, | |
| "logits/rejected": -3.0848946571350098, | |
| "logps/chosen": -0.7415497303009033, | |
| "logps/rejected": -0.982310950756073, | |
| "loss": 0.8005, | |
| "odds_ratio_loss": 0.589560866355896, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.07415496557950974, | |
| "rewards/margins": 0.024076132103800774, | |
| "rewards/rejected": -0.09823110699653625, | |
| "sft_loss": 0.7415497303009033, | |
| "step": 320 | |
| }, | |
| { | |
| "epoch": 0.533441099211962, | |
| "grad_norm": 1.4589684009552002, | |
| "learning_rate": 4.61940652694154e-06, | |
| "logits/chosen": -2.9786767959594727, | |
| "logits/rejected": -3.0444042682647705, | |
| "logps/chosen": -0.8274497985839844, | |
| "logps/rejected": -1.0422160625457764, | |
| "loss": 0.8954, | |
| "odds_ratio_loss": 0.6792970895767212, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08274497836828232, | |
| "rewards/margins": 0.021476630121469498, | |
| "rewards/rejected": -0.10422160476446152, | |
| "sft_loss": 0.8274497985839844, | |
| "step": 330 | |
| }, | |
| { | |
| "epoch": 0.5496059810062639, | |
| "grad_norm": 1.7346688508987427, | |
| "learning_rate": 4.596634117545689e-06, | |
| "logits/chosen": -3.0816709995269775, | |
| "logits/rejected": -3.0875000953674316, | |
| "logps/chosen": -0.8030570149421692, | |
| "logps/rejected": -1.0277900695800781, | |
| "loss": 0.8675, | |
| "odds_ratio_loss": 0.6442909836769104, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.08030570298433304, | |
| "rewards/margins": 0.022473318502306938, | |
| "rewards/rejected": -0.10277901589870453, | |
| "sft_loss": 0.8030570149421692, | |
| "step": 340 | |
| }, | |
| { | |
| "epoch": 0.5657708628005658, | |
| "grad_norm": 1.8591201305389404, | |
| "learning_rate": 4.573259648679335e-06, | |
| "logits/chosen": -3.0649001598358154, | |
| "logits/rejected": -3.029323101043701, | |
| "logps/chosen": -0.7972906231880188, | |
| "logps/rejected": -1.030458688735962, | |
| "loss": 0.8615, | |
| "odds_ratio_loss": 0.6421998739242554, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.07972906529903412, | |
| "rewards/margins": 0.02331680618226528, | |
| "rewards/rejected": -0.10304586589336395, | |
| "sft_loss": 0.7972906231880188, | |
| "step": 350 | |
| }, | |
| { | |
| "epoch": 0.5819357445948676, | |
| "grad_norm": 6.8644022941589355, | |
| "learning_rate": 4.549289832443663e-06, | |
| "logits/chosen": -3.0592093467712402, | |
| "logits/rejected": -3.0790326595306396, | |
| "logps/chosen": -0.7847403287887573, | |
| "logps/rejected": -1.0331987142562866, | |
| "loss": 0.8515, | |
| "odds_ratio_loss": 0.6680801510810852, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.0784740298986435, | |
| "rewards/margins": 0.024845842272043228, | |
| "rewards/rejected": -0.10331986844539642, | |
| "sft_loss": 0.7847403287887573, | |
| "step": 360 | |
| }, | |
| { | |
| "epoch": 0.5981006263891695, | |
| "grad_norm": 2.649265766143799, | |
| "learning_rate": 4.524731551896978e-06, | |
| "logits/chosen": -3.0282368659973145, | |
| "logits/rejected": -3.0504040718078613, | |
| "logps/chosen": -0.7369459271430969, | |
| "logps/rejected": -0.8965708017349243, | |
| "loss": 0.801, | |
| "odds_ratio_loss": 0.6401507258415222, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.07369460165500641, | |
| "rewards/margins": 0.015962477773427963, | |
| "rewards/rejected": -0.08965708315372467, | |
| "sft_loss": 0.7369459271430969, | |
| "step": 370 | |
| }, | |
| { | |
| "epoch": 0.6142655081834714, | |
| "grad_norm": 3.4245967864990234, | |
| "learning_rate": 4.4995918590781925e-06, | |
| "logits/chosen": -3.061760425567627, | |
| "logits/rejected": -3.0728158950805664, | |
| "logps/chosen": -0.7757102251052856, | |
| "logps/rejected": -0.9465000033378601, | |
| "loss": 0.8422, | |
| "odds_ratio_loss": 0.6651015281677246, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.07757101953029633, | |
| "rewards/margins": 0.017078977078199387, | |
| "rewards/rejected": -0.09464999288320541, | |
| "sft_loss": 0.7757102251052856, | |
| "step": 380 | |
| }, | |
| { | |
| "epoch": 0.6304303899777733, | |
| "grad_norm": 1.3795065879821777, | |
| "learning_rate": 4.473877972981797e-06, | |
| "logits/chosen": -3.0188069343566895, | |
| "logits/rejected": -3.024099588394165, | |
| "logps/chosen": -0.7883812189102173, | |
| "logps/rejected": -1.0172218084335327, | |
| "loss": 0.8495, | |
| "odds_ratio_loss": 0.6112133860588074, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.07883813232183456, | |
| "rewards/margins": 0.02288406528532505, | |
| "rewards/rejected": -0.10172219574451447, | |
| "sft_loss": 0.7883812189102173, | |
| "step": 390 | |
| }, | |
| { | |
| "epoch": 0.6465952717720752, | |
| "grad_norm": 1.795720100402832, | |
| "learning_rate": 4.447597277484894e-06, | |
| "logits/chosen": -2.9778666496276855, | |
| "logits/rejected": -3.0176868438720703, | |
| "logps/chosen": -0.743531346321106, | |
| "logps/rejected": -0.9195922017097473, | |
| "loss": 0.8054, | |
| "odds_ratio_loss": 0.6191025972366333, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.07435314357280731, | |
| "rewards/margins": 0.017606090754270554, | |
| "rewards/rejected": -0.09195923060178757, | |
| "sft_loss": 0.743531346321106, | |
| "step": 400 | |
| }, | |
| { | |
| "epoch": 0.6627601535663771, | |
| "grad_norm": 1.847349762916565, | |
| "learning_rate": 4.42075731922687e-06, | |
| "logits/chosen": -3.0738515853881836, | |
| "logits/rejected": -3.080390453338623, | |
| "logps/chosen": -0.8688371777534485, | |
| "logps/rejected": -1.0517194271087646, | |
| "loss": 0.9327, | |
| "odds_ratio_loss": 0.6381778120994568, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08688371628522873, | |
| "rewards/margins": 0.01828821375966072, | |
| "rewards/rejected": -0.10517191886901855, | |
| "sft_loss": 0.8688371777534485, | |
| "step": 410 | |
| }, | |
| { | |
| "epoch": 0.6789250353606789, | |
| "grad_norm": 5.425457954406738, | |
| "learning_rate": 4.3933658054423465e-06, | |
| "logits/chosen": -3.0373263359069824, | |
| "logits/rejected": -3.0470852851867676, | |
| "logps/chosen": -0.7733573913574219, | |
| "logps/rejected": -1.0374972820281982, | |
| "loss": 0.8332, | |
| "odds_ratio_loss": 0.5986987352371216, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.07733573764562607, | |
| "rewards/margins": 0.026413992047309875, | |
| "rewards/rejected": -0.10374973714351654, | |
| "sft_loss": 0.7733573913574219, | |
| "step": 420 | |
| }, | |
| { | |
| "epoch": 0.6950899171549808, | |
| "grad_norm": 2.5281944274902344, | |
| "learning_rate": 4.365430601748003e-06, | |
| "logits/chosen": -3.036982536315918, | |
| "logits/rejected": -3.0820653438568115, | |
| "logps/chosen": -0.8373786807060242, | |
| "logps/rejected": -0.9543370008468628, | |
| "loss": 0.9048, | |
| "odds_ratio_loss": 0.673850953578949, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.08373787254095078, | |
| "rewards/margins": 0.011695821769535542, | |
| "rewards/rejected": -0.09543369710445404, | |
| "sft_loss": 0.8373786807060242, | |
| "step": 430 | |
| }, | |
| { | |
| "epoch": 0.7112547989492827, | |
| "grad_norm": 6.225044250488281, | |
| "learning_rate": 4.336959729883925e-06, | |
| "logits/chosen": -3.0365397930145264, | |
| "logits/rejected": -3.063159942626953, | |
| "logps/chosen": -0.7677688598632812, | |
| "logps/rejected": -0.8747943043708801, | |
| "loss": 0.8394, | |
| "odds_ratio_loss": 0.7163954973220825, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.0767768919467926, | |
| "rewards/margins": 0.010702535510063171, | |
| "rewards/rejected": -0.08747942745685577, | |
| "sft_loss": 0.7677688598632812, | |
| "step": 440 | |
| }, | |
| { | |
| "epoch": 0.7274196807435845, | |
| "grad_norm": 4.238840103149414, | |
| "learning_rate": 4.307961365410118e-06, | |
| "logits/chosen": -3.031554698944092, | |
| "logits/rejected": -3.0537819862365723, | |
| "logps/chosen": -0.7852329015731812, | |
| "logps/rejected": -0.9492766261100769, | |
| "loss": 0.8479, | |
| "odds_ratio_loss": 0.6268799901008606, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.07852329313755035, | |
| "rewards/margins": 0.01640438288450241, | |
| "rewards/rejected": -0.09492767602205276, | |
| "sft_loss": 0.7852329015731812, | |
| "step": 450 | |
| }, | |
| { | |
| "epoch": 0.7435845625378864, | |
| "grad_norm": 3.3165500164031982, | |
| "learning_rate": 4.278443835358854e-06, | |
| "logits/chosen": -3.0518264770507812, | |
| "logits/rejected": -3.045757293701172, | |
| "logps/chosen": -0.7719421982765198, | |
| "logps/rejected": -1.0236573219299316, | |
| "loss": 0.831, | |
| "odds_ratio_loss": 0.5902360081672668, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.0771942138671875, | |
| "rewards/margins": 0.025171533226966858, | |
| "rewards/rejected": -0.10236574709415436, | |
| "sft_loss": 0.7719421982765198, | |
| "step": 460 | |
| }, | |
| { | |
| "epoch": 0.7597494443321883, | |
| "grad_norm": 2.330195426940918, | |
| "learning_rate": 4.248415615843523e-06, | |
| "logits/chosen": -3.079732656478882, | |
| "logits/rejected": -3.0858983993530273, | |
| "logps/chosen": -0.7835872769355774, | |
| "logps/rejected": -0.9195672273635864, | |
| "loss": 0.8504, | |
| "odds_ratio_loss": 0.6679321527481079, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.0783587247133255, | |
| "rewards/margins": 0.01359798014163971, | |
| "rewards/rejected": -0.0919567197561264, | |
| "sft_loss": 0.7835872769355774, | |
| "step": 470 | |
| }, | |
| { | |
| "epoch": 0.7759143261264903, | |
| "grad_norm": 6.524634838104248, | |
| "learning_rate": 4.217885329624666e-06, | |
| "logits/chosen": -3.0687060356140137, | |
| "logits/rejected": -3.066584348678589, | |
| "logps/chosen": -0.7517032623291016, | |
| "logps/rejected": -0.9625965356826782, | |
| "loss": 0.8127, | |
| "odds_ratio_loss": 0.6098276376724243, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.07517032325267792, | |
| "rewards/margins": 0.021089335903525352, | |
| "rewards/rejected": -0.09625966101884842, | |
| "sft_loss": 0.7517032623291016, | |
| "step": 480 | |
| }, | |
| { | |
| "epoch": 0.7920792079207921, | |
| "grad_norm": 3.629946231842041, | |
| "learning_rate": 4.186861743633911e-06, | |
| "logits/chosen": -3.0519471168518066, | |
| "logits/rejected": -3.0878560543060303, | |
| "logps/chosen": -0.7691044807434082, | |
| "logps/rejected": -1.0053441524505615, | |
| "loss": 0.8348, | |
| "odds_ratio_loss": 0.6568228006362915, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.07691045105457306, | |
| "rewards/margins": 0.023623958230018616, | |
| "rewards/rejected": -0.10053440183401108, | |
| "sft_loss": 0.7691044807434082, | |
| "step": 490 | |
| }, | |
| { | |
| "epoch": 0.808244089715094, | |
| "grad_norm": 1.9105418920516968, | |
| "learning_rate": 4.155353766456497e-06, | |
| "logits/chosen": -3.1221230030059814, | |
| "logits/rejected": -3.1099045276641846, | |
| "logps/chosen": -0.8151519894599915, | |
| "logps/rejected": -0.9320052862167358, | |
| "loss": 0.8803, | |
| "odds_ratio_loss": 0.6510958671569824, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.08151519298553467, | |
| "rewards/margins": 0.011685335077345371, | |
| "rewards/rejected": -0.09320052713155746, | |
| "sft_loss": 0.8151519894599915, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.808244089715094, | |
| "eval_logits/chosen": -3.052025556564331, | |
| "eval_logits/rejected": -3.0746355056762695, | |
| "eval_logps/chosen": -0.7961810827255249, | |
| "eval_logps/rejected": -0.9834145307540894, | |
| "eval_loss": 0.8619003891944885, | |
| "eval_odds_ratio_loss": 0.6571925282478333, | |
| "eval_rewards/accuracies": 0.5654545426368713, | |
| "eval_rewards/chosen": -0.07961811125278473, | |
| "eval_rewards/margins": 0.018723346292972565, | |
| "eval_rewards/rejected": -0.0983414575457573, | |
| "eval_runtime": 369.3504, | |
| "eval_samples_per_second": 2.978, | |
| "eval_sft_loss": 0.7961810827255249, | |
| "eval_steps_per_second": 1.489, | |
| "step": 500 | |
| }, | |
| { | |
| "epoch": 0.8244089715093958, | |
| "grad_norm": 2.1485707759857178, | |
| "learning_rate": 4.123370445773134e-06, | |
| "logits/chosen": -3.0945208072662354, | |
| "logits/rejected": -3.1061959266662598, | |
| "logps/chosen": -0.788953423500061, | |
| "logps/rejected": -0.8694869875907898, | |
| "loss": 0.8606, | |
| "odds_ratio_loss": 0.7168292999267578, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.07889534533023834, | |
| "rewards/margins": 0.008053350262343884, | |
| "rewards/rejected": -0.0869487002491951, | |
| "sft_loss": 0.788953423500061, | |
| "step": 510 | |
| }, | |
| { | |
| "epoch": 0.8405738533036977, | |
| "grad_norm": 5.462285041809082, | |
| "learning_rate": 4.090920965761906e-06, | |
| "logits/chosen": -3.0212631225585938, | |
| "logits/rejected": -3.031066417694092, | |
| "logps/chosen": -0.8095367550849915, | |
| "logps/rejected": -0.968643844127655, | |
| "loss": 0.8752, | |
| "odds_ratio_loss": 0.656153678894043, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.0809536725282669, | |
| "rewards/margins": 0.015910711139440536, | |
| "rewards/rejected": -0.09686438739299774, | |
| "sft_loss": 0.8095367550849915, | |
| "step": 520 | |
| }, | |
| { | |
| "epoch": 0.8567387350979996, | |
| "grad_norm": 4.947661399841309, | |
| "learning_rate": 4.058014644460991e-06, | |
| "logits/chosen": -3.0334737300872803, | |
| "logits/rejected": -3.049567937850952, | |
| "logps/chosen": -0.756322979927063, | |
| "logps/rejected": -0.8999664187431335, | |
| "loss": 0.8179, | |
| "odds_ratio_loss": 0.6152733564376831, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.07563230395317078, | |
| "rewards/margins": 0.014364344999194145, | |
| "rewards/rejected": -0.08999665081501007, | |
| "sft_loss": 0.756322979927063, | |
| "step": 530 | |
| }, | |
| { | |
| "epoch": 0.8729036168923014, | |
| "grad_norm": 2.2244179248809814, | |
| "learning_rate": 4.024660931092939e-06, | |
| "logits/chosen": -3.0092921257019043, | |
| "logits/rejected": -3.0213623046875, | |
| "logps/chosen": -0.7882963418960571, | |
| "logps/rejected": -0.9846014976501465, | |
| "loss": 0.8497, | |
| "odds_ratio_loss": 0.6144701838493347, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.07882963120937347, | |
| "rewards/margins": 0.019630510360002518, | |
| "rewards/rejected": -0.0984601378440857, | |
| "sft_loss": 0.7882963418960571, | |
| "step": 540 | |
| }, | |
| { | |
| "epoch": 0.8890684986866033, | |
| "grad_norm": 1.6228642463684082, | |
| "learning_rate": 3.990869403351272e-06, | |
| "logits/chosen": -3.051035165786743, | |
| "logits/rejected": -3.073690176010132, | |
| "logps/chosen": -0.794242262840271, | |
| "logps/rejected": -1.0482288599014282, | |
| "loss": 0.8507, | |
| "odds_ratio_loss": 0.564966082572937, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.07942423224449158, | |
| "rewards/margins": 0.025398656725883484, | |
| "rewards/rejected": -0.10482288897037506, | |
| "sft_loss": 0.794242262840271, | |
| "step": 550 | |
| }, | |
| { | |
| "epoch": 0.9052333804809052, | |
| "grad_norm": 3.156888008117676, | |
| "learning_rate": 3.956649764650206e-06, | |
| "logits/chosen": -3.1107125282287598, | |
| "logits/rejected": -3.121273994445801, | |
| "logps/chosen": -0.7970541715621948, | |
| "logps/rejected": -0.9795141220092773, | |
| "loss": 0.864, | |
| "odds_ratio_loss": 0.6697722673416138, | |
| "rewards/accuracies": 0.4625000059604645, | |
| "rewards/chosen": -0.07970540225505829, | |
| "rewards/margins": 0.018245995044708252, | |
| "rewards/rejected": -0.09795141220092773, | |
| "sft_loss": 0.7970541715621948, | |
| "step": 560 | |
| }, | |
| { | |
| "epoch": 0.9213982622752072, | |
| "grad_norm": 4.841604232788086, | |
| "learning_rate": 3.92201184133826e-06, | |
| "logits/chosen": -3.082477331161499, | |
| "logits/rejected": -3.0972495079040527, | |
| "logps/chosen": -0.7547809481620789, | |
| "logps/rejected": -0.9688172340393066, | |
| "loss": 0.8159, | |
| "odds_ratio_loss": 0.6109867095947266, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.07547809928655624, | |
| "rewards/margins": 0.021403620019555092, | |
| "rewards/rejected": -0.09688171744346619, | |
| "sft_loss": 0.7547809481620789, | |
| "step": 570 | |
| }, | |
| { | |
| "epoch": 0.937563144069509, | |
| "grad_norm": 2.3844194412231445, | |
| "learning_rate": 3.886965579876572e-06, | |
| "logits/chosen": -3.114271640777588, | |
| "logits/rejected": -3.1176934242248535, | |
| "logps/chosen": -0.7549653053283691, | |
| "logps/rejected": -0.858431339263916, | |
| "loss": 0.8214, | |
| "odds_ratio_loss": 0.664365828037262, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.07549652457237244, | |
| "rewards/margins": 0.010346608236432076, | |
| "rewards/rejected": -0.08584313094615936, | |
| "sft_loss": 0.7549653053283691, | |
| "step": 580 | |
| }, | |
| { | |
| "epoch": 0.9537280258638109, | |
| "grad_norm": 1.683606743812561, | |
| "learning_rate": 3.851521043982716e-06, | |
| "logits/chosen": -3.1139140129089355, | |
| "logits/rejected": -3.134669780731201, | |
| "logps/chosen": -0.793258011341095, | |
| "logps/rejected": -0.9314570426940918, | |
| "loss": 0.8603, | |
| "odds_ratio_loss": 0.670341432094574, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.07932581007480621, | |
| "rewards/margins": 0.013819903135299683, | |
| "rewards/rejected": -0.0931456983089447, | |
| "sft_loss": 0.793258011341095, | |
| "step": 590 | |
| }, | |
| { | |
| "epoch": 0.9698929076581128, | |
| "grad_norm": 6.090215682983398, | |
| "learning_rate": 3.81568841174086e-06, | |
| "logits/chosen": -3.0772290229797363, | |
| "logits/rejected": -3.1182470321655273, | |
| "logps/chosen": -0.7526463270187378, | |
| "logps/rejected": -0.9828107953071594, | |
| "loss": 0.8172, | |
| "odds_ratio_loss": 0.6451513171195984, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.07526463270187378, | |
| "rewards/margins": 0.023016449064016342, | |
| "rewards/rejected": -0.09828107804059982, | |
| "sft_loss": 0.7526463270187378, | |
| "step": 600 | |
| }, | |
| { | |
| "epoch": 0.9860577894524146, | |
| "grad_norm": 1.9422472715377808, | |
| "learning_rate": 3.7794779726790664e-06, | |
| "logits/chosen": -3.064382314682007, | |
| "logits/rejected": -3.09804105758667, | |
| "logps/chosen": -0.736395537853241, | |
| "logps/rejected": -0.9441590309143066, | |
| "loss": 0.798, | |
| "odds_ratio_loss": 0.6159018278121948, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.07363955676555634, | |
| "rewards/margins": 0.02077634632587433, | |
| "rewards/rejected": -0.09441590309143066, | |
| "sft_loss": 0.736395537853241, | |
| "step": 610 | |
| }, | |
| { | |
| "epoch": 1.0022226712467166, | |
| "grad_norm": 3.0195484161376953, | |
| "learning_rate": 3.7429001248146096e-06, | |
| "logits/chosen": -3.0517828464508057, | |
| "logits/rejected": -3.0764060020446777, | |
| "logps/chosen": -0.7535146474838257, | |
| "logps/rejected": -0.9733519554138184, | |
| "loss": 0.8118, | |
| "odds_ratio_loss": 0.582946240901947, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.07535146921873093, | |
| "rewards/margins": 0.02198372781276703, | |
| "rewards/rejected": -0.09733519703149796, | |
| "sft_loss": 0.7535146474838257, | |
| "step": 620 | |
| }, | |
| { | |
| "epoch": 1.0183875530410185, | |
| "grad_norm": 1.4307211637496948, | |
| "learning_rate": 3.7059653716681227e-06, | |
| "logits/chosen": -3.0821685791015625, | |
| "logits/rejected": -3.0729098320007324, | |
| "logps/chosen": -0.8158019185066223, | |
| "logps/rejected": -1.0031102895736694, | |
| "loss": 0.8821, | |
| "odds_ratio_loss": 0.6627525091171265, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.08158019185066223, | |
| "rewards/margins": 0.01873084530234337, | |
| "rewards/rejected": -0.1003110408782959, | |
| "sft_loss": 0.8158019185066223, | |
| "step": 630 | |
| }, | |
| { | |
| "epoch": 1.0345524348353203, | |
| "grad_norm": 1.925041675567627, | |
| "learning_rate": 3.668684319247463e-06, | |
| "logits/chosen": -3.11029052734375, | |
| "logits/rejected": -3.133237600326538, | |
| "logps/chosen": -0.7127649188041687, | |
| "logps/rejected": -0.9635750651359558, | |
| "loss": 0.7692, | |
| "odds_ratio_loss": 0.5642341375350952, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.07127650082111359, | |
| "rewards/margins": 0.02508101798593998, | |
| "rewards/rejected": -0.09635750949382782, | |
| "sft_loss": 0.7127649188041687, | |
| "step": 640 | |
| }, | |
| { | |
| "epoch": 1.0507173166296222, | |
| "grad_norm": 1.3039898872375488, | |
| "learning_rate": 3.6310676730021373e-06, | |
| "logits/chosen": -3.1486330032348633, | |
| "logits/rejected": -3.1563305854797363, | |
| "logps/chosen": -0.7767339944839478, | |
| "logps/rejected": -0.9257136583328247, | |
| "loss": 0.8402, | |
| "odds_ratio_loss": 0.6346315145492554, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.07767340540885925, | |
| "rewards/margins": 0.014897963032126427, | |
| "rewards/rejected": -0.09257137030363083, | |
| "sft_loss": 0.7767339944839478, | |
| "step": 650 | |
| }, | |
| { | |
| "epoch": 1.066882198423924, | |
| "grad_norm": 8.102341651916504, | |
| "learning_rate": 3.593126234749178e-06, | |
| "logits/chosen": -3.1005399227142334, | |
| "logits/rejected": -3.1532058715820312, | |
| "logps/chosen": -0.8182880282402039, | |
| "logps/rejected": -0.9324063062667847, | |
| "loss": 0.8843, | |
| "odds_ratio_loss": 0.6604392528533936, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.08182881772518158, | |
| "rewards/margins": 0.011411817744374275, | |
| "rewards/rejected": -0.0932406336069107, | |
| "sft_loss": 0.8182880282402039, | |
| "step": 660 | |
| }, | |
| { | |
| "epoch": 1.083047080218226, | |
| "grad_norm": 2.537015438079834, | |
| "learning_rate": 3.554870899571343e-06, | |
| "logits/chosen": -3.116610050201416, | |
| "logits/rejected": -3.1441800594329834, | |
| "logps/chosen": -0.7676142454147339, | |
| "logps/rejected": -0.9440106153488159, | |
| "loss": 0.8313, | |
| "odds_ratio_loss": 0.6365249156951904, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.07676141709089279, | |
| "rewards/margins": 0.0176396407186985, | |
| "rewards/rejected": -0.09440106153488159, | |
| "sft_loss": 0.7676142454147339, | |
| "step": 670 | |
| }, | |
| { | |
| "epoch": 1.0992119620125278, | |
| "grad_norm": 6.732462406158447, | |
| "learning_rate": 3.5163126526885373e-06, | |
| "logits/chosen": -3.077462673187256, | |
| "logits/rejected": -3.1246635913848877, | |
| "logps/chosen": -0.7416545152664185, | |
| "logps/rejected": -0.9575145840644836, | |
| "loss": 0.8024, | |
| "odds_ratio_loss": 0.607850193977356, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.0741654485464096, | |
| "rewards/margins": 0.02158600650727749, | |
| "rewards/rejected": -0.09575144946575165, | |
| "sft_loss": 0.7416545152664185, | |
| "step": 680 | |
| }, | |
| { | |
| "epoch": 1.1153768438068297, | |
| "grad_norm": 1.995318055152893, | |
| "learning_rate": 3.4774625663033484e-06, | |
| "logits/chosen": -3.088719367980957, | |
| "logits/rejected": -3.1156630516052246, | |
| "logps/chosen": -0.7460827231407166, | |
| "logps/rejected": -0.913016140460968, | |
| "loss": 0.8092, | |
| "odds_ratio_loss": 0.6309585571289062, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.07460827380418777, | |
| "rewards/margins": 0.01669333688914776, | |
| "rewards/rejected": -0.09130160510540009, | |
| "sft_loss": 0.7460827231407166, | |
| "step": 690 | |
| }, | |
| { | |
| "epoch": 1.1315417256011315, | |
| "grad_norm": 1.8286596536636353, | |
| "learning_rate": 3.4383317964216067e-06, | |
| "logits/chosen": -3.0861454010009766, | |
| "logits/rejected": -3.1318280696868896, | |
| "logps/chosen": -0.7198506593704224, | |
| "logps/rejected": -0.8674876093864441, | |
| "loss": 0.7881, | |
| "odds_ratio_loss": 0.6827356219291687, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.07198506593704224, | |
| "rewards/margins": 0.014763685874640942, | |
| "rewards/rejected": -0.08674876391887665, | |
| "sft_loss": 0.7198506593704224, | |
| "step": 700 | |
| }, | |
| { | |
| "epoch": 1.1477066073954334, | |
| "grad_norm": 3.1164824962615967, | |
| "learning_rate": 3.398931579648877e-06, | |
| "logits/chosen": -3.1039557456970215, | |
| "logits/rejected": -3.141571521759033, | |
| "logps/chosen": -0.7915211915969849, | |
| "logps/rejected": -1.108933687210083, | |
| "loss": 0.8534, | |
| "odds_ratio_loss": 0.6187322735786438, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.07915211468935013, | |
| "rewards/margins": 0.031741250306367874, | |
| "rewards/rejected": -0.1108933687210083, | |
| "sft_loss": 0.7915211915969849, | |
| "step": 710 | |
| }, | |
| { | |
| "epoch": 1.1638714891897353, | |
| "grad_norm": 1.4664254188537598, | |
| "learning_rate": 3.359273229963813e-06, | |
| "logits/chosen": -3.1003873348236084, | |
| "logits/rejected": -3.103529214859009, | |
| "logps/chosen": -0.730610191822052, | |
| "logps/rejected": -0.8717561960220337, | |
| "loss": 0.7957, | |
| "odds_ratio_loss": 0.6507007479667664, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.0730610117316246, | |
| "rewards/margins": 0.014114594087004662, | |
| "rewards/rejected": -0.08717561513185501, | |
| "sft_loss": 0.730610191822052, | |
| "step": 720 | |
| }, | |
| { | |
| "epoch": 1.1800363709840371, | |
| "grad_norm": 1.3891639709472656, | |
| "learning_rate": 3.319368135469285e-06, | |
| "logits/chosen": -3.1091885566711426, | |
| "logits/rejected": -3.150235652923584, | |
| "logps/chosen": -0.7716542482376099, | |
| "logps/rejected": -1.0244488716125488, | |
| "loss": 0.837, | |
| "odds_ratio_loss": 0.6535542011260986, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.07716542482376099, | |
| "rewards/margins": 0.025279458612203598, | |
| "rewards/rejected": -0.10244487226009369, | |
| "sft_loss": 0.7716542482376099, | |
| "step": 730 | |
| }, | |
| { | |
| "epoch": 1.196201252778339, | |
| "grad_norm": 3.498404026031494, | |
| "learning_rate": 3.279227755122228e-06, | |
| "logits/chosen": -3.0913896560668945, | |
| "logits/rejected": -3.114501476287842, | |
| "logps/chosen": -0.7233365178108215, | |
| "logps/rejected": -1.0984933376312256, | |
| "loss": 0.778, | |
| "odds_ratio_loss": 0.5463576912879944, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -0.07233365625143051, | |
| "rewards/margins": 0.037515684962272644, | |
| "rewards/rejected": -0.10984933376312256, | |
| "sft_loss": 0.7233365178108215, | |
| "step": 740 | |
| }, | |
| { | |
| "epoch": 1.2123661345726409, | |
| "grad_norm": 1.6691545248031616, | |
| "learning_rate": 3.2388636154431417e-06, | |
| "logits/chosen": -3.1425840854644775, | |
| "logits/rejected": -3.175088405609131, | |
| "logps/chosen": -0.7860497236251831, | |
| "logps/rejected": -1.0689184665679932, | |
| "loss": 0.846, | |
| "odds_ratio_loss": 0.5996078252792358, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.07860498130321503, | |
| "rewards/margins": 0.028286874294281006, | |
| "rewards/rejected": -0.10689185559749603, | |
| "sft_loss": 0.7860497236251831, | |
| "step": 750 | |
| }, | |
| { | |
| "epoch": 1.2285310163669427, | |
| "grad_norm": 1.6291383504867554, | |
| "learning_rate": 3.198287307206192e-06, | |
| "logits/chosen": -3.0734639167785645, | |
| "logits/rejected": -3.0943312644958496, | |
| "logps/chosen": -0.7356687784194946, | |
| "logps/rejected": -0.9488394856452942, | |
| "loss": 0.7952, | |
| "odds_ratio_loss": 0.5951117277145386, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.07356687635183334, | |
| "rewards/margins": 0.021317074075341225, | |
| "rewards/rejected": -0.09488394856452942, | |
| "sft_loss": 0.7356687784194946, | |
| "step": 760 | |
| }, | |
| { | |
| "epoch": 1.2446958981612446, | |
| "grad_norm": 1.6178934574127197, | |
| "learning_rate": 3.157510482110856e-06, | |
| "logits/chosen": -3.143188238143921, | |
| "logits/rejected": -3.1431174278259277, | |
| "logps/chosen": -0.7597110271453857, | |
| "logps/rejected": -0.9578113555908203, | |
| "loss": 0.8251, | |
| "odds_ratio_loss": 0.6543140411376953, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.0759711042046547, | |
| "rewards/margins": 0.019810039550065994, | |
| "rewards/rejected": -0.09578114002943039, | |
| "sft_loss": 0.7597110271453857, | |
| "step": 770 | |
| }, | |
| { | |
| "epoch": 1.2608607799555465, | |
| "grad_norm": 1.6571840047836304, | |
| "learning_rate": 3.116544849436077e-06, | |
| "logits/chosen": -3.084550142288208, | |
| "logits/rejected": -3.089481830596924, | |
| "logps/chosen": -0.804486095905304, | |
| "logps/rejected": -1.1236674785614014, | |
| "loss": 0.8661, | |
| "odds_ratio_loss": 0.6160328388214111, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.08044860512018204, | |
| "rewards/margins": 0.03191814199090004, | |
| "rewards/rejected": -0.11236675083637238, | |
| "sft_loss": 0.804486095905304, | |
| "step": 780 | |
| }, | |
| { | |
| "epoch": 1.2770256617498483, | |
| "grad_norm": 2.3182828426361084, | |
| "learning_rate": 3.0754021726778848e-06, | |
| "logits/chosen": -3.086887836456299, | |
| "logits/rejected": -3.093517780303955, | |
| "logps/chosen": -0.7223183512687683, | |
| "logps/rejected": -1.0067367553710938, | |
| "loss": 0.7785, | |
| "odds_ratio_loss": 0.5618979334831238, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -0.07223184406757355, | |
| "rewards/margins": 0.028441840782761574, | |
| "rewards/rejected": -0.10067367553710938, | |
| "sft_loss": 0.7223183512687683, | |
| "step": 790 | |
| }, | |
| { | |
| "epoch": 1.2931905435441502, | |
| "grad_norm": 1.3017044067382812, | |
| "learning_rate": 3.0340942661714463e-06, | |
| "logits/chosen": -3.1351680755615234, | |
| "logits/rejected": -3.134371280670166, | |
| "logps/chosen": -0.796164333820343, | |
| "logps/rejected": -0.9728119969367981, | |
| "loss": 0.8595, | |
| "odds_ratio_loss": 0.6329259276390076, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.07961643487215042, | |
| "rewards/margins": 0.017664765939116478, | |
| "rewards/rejected": -0.09728120267391205, | |
| "sft_loss": 0.796164333820343, | |
| "step": 800 | |
| }, | |
| { | |
| "epoch": 1.3093554253384523, | |
| "grad_norm": 2.427229166030884, | |
| "learning_rate": 2.992632991698512e-06, | |
| "logits/chosen": -3.082369327545166, | |
| "logits/rejected": -3.110403060913086, | |
| "logps/chosen": -0.7753532528877258, | |
| "logps/rejected": -0.9790387153625488, | |
| "loss": 0.8377, | |
| "odds_ratio_loss": 0.6236809492111206, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.07753531634807587, | |
| "rewards/margins": 0.0203685499727726, | |
| "rewards/rejected": -0.09790387749671936, | |
| "sft_loss": 0.7753532528877258, | |
| "step": 810 | |
| }, | |
| { | |
| "epoch": 1.3255203071327541, | |
| "grad_norm": 1.9767731428146362, | |
| "learning_rate": 2.9510302550812537e-06, | |
| "logits/chosen": -3.089890480041504, | |
| "logits/rejected": -3.138611078262329, | |
| "logps/chosen": -0.6810489892959595, | |
| "logps/rejected": -1.002256155014038, | |
| "loss": 0.7382, | |
| "odds_ratio_loss": 0.571026623249054, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.06810488551855087, | |
| "rewards/margins": 0.032120734453201294, | |
| "rewards/rejected": -0.10022562742233276, | |
| "sft_loss": 0.6810489892959595, | |
| "step": 820 | |
| }, | |
| { | |
| "epoch": 1.341685188927056, | |
| "grad_norm": 4.014182090759277, | |
| "learning_rate": 2.9092980027634325e-06, | |
| "logits/chosen": -3.095803737640381, | |
| "logits/rejected": -3.1307053565979004, | |
| "logps/chosen": -0.7113646268844604, | |
| "logps/rejected": -0.9739853143692017, | |
| "loss": 0.773, | |
| "odds_ratio_loss": 0.6160944700241089, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.0711364597082138, | |
| "rewards/margins": 0.02626206912100315, | |
| "rewards/rejected": -0.0973985344171524, | |
| "sft_loss": 0.7113646268844604, | |
| "step": 830 | |
| }, | |
| { | |
| "epoch": 1.3578500707213579, | |
| "grad_norm": 2.0923309326171875, | |
| "learning_rate": 2.867448218379927e-06, | |
| "logits/chosen": -3.092233419418335, | |
| "logits/rejected": -3.1379733085632324, | |
| "logps/chosen": -0.8104713559150696, | |
| "logps/rejected": -0.9551480412483215, | |
| "loss": 0.8783, | |
| "odds_ratio_loss": 0.6786799430847168, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.08104713261127472, | |
| "rewards/margins": 0.014467673376202583, | |
| "rewards/rejected": -0.09551481157541275, | |
| "sft_loss": 0.8104713559150696, | |
| "step": 840 | |
| }, | |
| { | |
| "epoch": 1.3740149525156597, | |
| "grad_norm": 5.772988319396973, | |
| "learning_rate": 2.825492919315559e-06, | |
| "logits/chosen": -3.1081454753875732, | |
| "logits/rejected": -3.1527466773986816, | |
| "logps/chosen": -0.8419367671012878, | |
| "logps/rejected": -0.9567692875862122, | |
| "loss": 0.9094, | |
| "odds_ratio_loss": 0.6746194958686829, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.08419367671012878, | |
| "rewards/margins": 0.011483250185847282, | |
| "rewards/rejected": -0.09567694365978241, | |
| "sft_loss": 0.8419367671012878, | |
| "step": 850 | |
| }, | |
| { | |
| "epoch": 1.3901798343099616, | |
| "grad_norm": 3.8201489448547363, | |
| "learning_rate": 2.7834441532542482e-06, | |
| "logits/chosen": -3.1463775634765625, | |
| "logits/rejected": -3.1600661277770996, | |
| "logps/chosen": -0.7513538599014282, | |
| "logps/rejected": -0.964932918548584, | |
| "loss": 0.8093, | |
| "odds_ratio_loss": 0.579144299030304, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.07513538748025894, | |
| "rewards/margins": 0.021357912570238113, | |
| "rewards/rejected": -0.09649328887462616, | |
| "sft_loss": 0.7513538599014282, | |
| "step": 860 | |
| }, | |
| { | |
| "epoch": 1.4063447161042635, | |
| "grad_norm": 2.0883562564849854, | |
| "learning_rate": 2.74131399471945e-06, | |
| "logits/chosen": -3.1237385272979736, | |
| "logits/rejected": -3.1525537967681885, | |
| "logps/chosen": -0.7669543027877808, | |
| "logps/rejected": -0.9222270250320435, | |
| "loss": 0.8335, | |
| "odds_ratio_loss": 0.6657834053039551, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.07669542729854584, | |
| "rewards/margins": 0.0155272725969553, | |
| "rewards/rejected": -0.09222270548343658, | |
| "sft_loss": 0.7669543027877808, | |
| "step": 870 | |
| }, | |
| { | |
| "epoch": 1.4225095978985653, | |
| "grad_norm": 2.7775161266326904, | |
| "learning_rate": 2.6991145416068947e-06, | |
| "logits/chosen": -3.078185796737671, | |
| "logits/rejected": -3.1287853717803955, | |
| "logps/chosen": -0.7748882174491882, | |
| "logps/rejected": -0.8965023159980774, | |
| "loss": 0.8403, | |
| "odds_ratio_loss": 0.6543062925338745, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.07748880982398987, | |
| "rewards/margins": 0.012161416001617908, | |
| "rewards/rejected": -0.0896502360701561, | |
| "sft_loss": 0.7748882174491882, | |
| "step": 880 | |
| }, | |
| { | |
| "epoch": 1.4386744796928672, | |
| "grad_norm": 1.0541467666625977, | |
| "learning_rate": 2.6568579117106143e-06, | |
| "logits/chosen": -3.077782392501831, | |
| "logits/rejected": -3.0923101902008057, | |
| "logps/chosen": -0.7235719561576843, | |
| "logps/rejected": -0.9604678153991699, | |
| "loss": 0.7832, | |
| "odds_ratio_loss": 0.5962098240852356, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.07235720008611679, | |
| "rewards/margins": 0.02368958666920662, | |
| "rewards/rejected": -0.09604678303003311, | |
| "sft_loss": 0.7235719561576843, | |
| "step": 890 | |
| }, | |
| { | |
| "epoch": 1.454839361487169, | |
| "grad_norm": 1.1944150924682617, | |
| "learning_rate": 2.6145562392432544e-06, | |
| "logits/chosen": -3.139462947845459, | |
| "logits/rejected": -3.1433637142181396, | |
| "logps/chosen": -0.7596802115440369, | |
| "logps/rejected": -0.9312537312507629, | |
| "loss": 0.8259, | |
| "odds_ratio_loss": 0.6623841524124146, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.07596802711486816, | |
| "rewards/margins": 0.01715734973549843, | |
| "rewards/rejected": -0.09312538057565689, | |
| "sft_loss": 0.7596802115440369, | |
| "step": 900 | |
| }, | |
| { | |
| "epoch": 1.471004243281471, | |
| "grad_norm": 3.4694266319274902, | |
| "learning_rate": 2.5722216713516682e-06, | |
| "logits/chosen": -3.0813755989074707, | |
| "logits/rejected": -3.128629207611084, | |
| "logps/chosen": -0.7107754945755005, | |
| "logps/rejected": -0.9305357933044434, | |
| "loss": 0.7723, | |
| "odds_ratio_loss": 0.6156551837921143, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.07107754051685333, | |
| "rewards/margins": 0.021976038813591003, | |
| "rewards/rejected": -0.09305357933044434, | |
| "sft_loss": 0.7107754945755005, | |
| "step": 910 | |
| }, | |
| { | |
| "epoch": 1.4871691250757728, | |
| "grad_norm": 4.237662315368652, | |
| "learning_rate": 2.5298663646288064e-06, | |
| "logits/chosen": -3.1253597736358643, | |
| "logits/rejected": -3.1492881774902344, | |
| "logps/chosen": -0.7484380602836609, | |
| "logps/rejected": -0.9847918748855591, | |
| "loss": 0.8097, | |
| "odds_ratio_loss": 0.6130812168121338, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.07484380900859833, | |
| "rewards/margins": 0.023635383695364, | |
| "rewards/rejected": -0.09847918897867203, | |
| "sft_loss": 0.7484380602836609, | |
| "step": 920 | |
| }, | |
| { | |
| "epoch": 1.503334006870075, | |
| "grad_norm": 2.1491661071777344, | |
| "learning_rate": 2.487502481622879e-06, | |
| "logits/chosen": -3.0927679538726807, | |
| "logits/rejected": -3.110395908355713, | |
| "logps/chosen": -0.7894285321235657, | |
| "logps/rejected": -0.9456027150154114, | |
| "loss": 0.8507, | |
| "odds_ratio_loss": 0.6129187345504761, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.07894285023212433, | |
| "rewards/margins": 0.01561742089688778, | |
| "rewards/rejected": -0.09456028044223785, | |
| "sft_loss": 0.7894285321235657, | |
| "step": 930 | |
| }, | |
| { | |
| "epoch": 1.5194988886643768, | |
| "grad_norm": 1.948228359222412, | |
| "learning_rate": 2.4451421873448253e-06, | |
| "logits/chosen": -3.0993218421936035, | |
| "logits/rejected": -3.1382908821105957, | |
| "logps/chosen": -0.7644907832145691, | |
| "logps/rejected": -0.9289671778678894, | |
| "loss": 0.8309, | |
| "odds_ratio_loss": 0.6645576357841492, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.07644907385110855, | |
| "rewards/margins": 0.016447637230157852, | |
| "rewards/rejected": -0.0928967148065567, | |
| "sft_loss": 0.7644907832145691, | |
| "step": 940 | |
| }, | |
| { | |
| "epoch": 1.5356637704586786, | |
| "grad_norm": 5.180652141571045, | |
| "learning_rate": 2.40279764577506e-06, | |
| "logits/chosen": -3.1537530422210693, | |
| "logits/rejected": -3.1876587867736816, | |
| "logps/chosen": -0.7837322354316711, | |
| "logps/rejected": -0.9118620157241821, | |
| "loss": 0.8488, | |
| "odds_ratio_loss": 0.6507243514060974, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.07837323099374771, | |
| "rewards/margins": 0.012812974862754345, | |
| "rewards/rejected": -0.09118620306253433, | |
| "sft_loss": 0.7837322354316711, | |
| "step": 950 | |
| }, | |
| { | |
| "epoch": 1.5518286522529805, | |
| "grad_norm": 3.1721439361572266, | |
| "learning_rate": 2.3604810163705242e-06, | |
| "logits/chosen": -3.117772102355957, | |
| "logits/rejected": -3.1582953929901123, | |
| "logps/chosen": -0.6980777978897095, | |
| "logps/rejected": -0.9280182719230652, | |
| "loss": 0.756, | |
| "odds_ratio_loss": 0.5792473554611206, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -0.06980777531862259, | |
| "rewards/margins": 0.02299405261874199, | |
| "rewards/rejected": -0.09280182421207428, | |
| "sft_loss": 0.6980777978897095, | |
| "step": 960 | |
| }, | |
| { | |
| "epoch": 1.5679935340472824, | |
| "grad_norm": 2.930034637451172, | |
| "learning_rate": 2.3182044505730364e-06, | |
| "logits/chosen": -3.102292537689209, | |
| "logits/rejected": -3.1203739643096924, | |
| "logps/chosen": -0.6645774245262146, | |
| "logps/rejected": -0.8846859931945801, | |
| "loss": 0.7224, | |
| "odds_ratio_loss": 0.5782453417778015, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.06645774096250534, | |
| "rewards/margins": 0.02201084978878498, | |
| "rewards/rejected": -0.08846859633922577, | |
| "sft_loss": 0.6645774245262146, | |
| "step": 970 | |
| }, | |
| { | |
| "epoch": 1.5841584158415842, | |
| "grad_norm": 4.4542083740234375, | |
| "learning_rate": 2.275980088319941e-06, | |
| "logits/chosen": -3.1215856075286865, | |
| "logits/rejected": -3.1170594692230225, | |
| "logps/chosen": -0.718813955783844, | |
| "logps/rejected": -0.8550910949707031, | |
| "loss": 0.7869, | |
| "odds_ratio_loss": 0.6806875467300415, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.07188138365745544, | |
| "rewards/margins": 0.013627724722027779, | |
| "rewards/rejected": -0.08550911396741867, | |
| "sft_loss": 0.718813955783844, | |
| "step": 980 | |
| }, | |
| { | |
| "epoch": 1.600323297635886, | |
| "grad_norm": 3.0541982650756836, | |
| "learning_rate": 2.2338200545580577e-06, | |
| "logits/chosen": -3.0836963653564453, | |
| "logits/rejected": -3.1206369400024414, | |
| "logps/chosen": -0.710333526134491, | |
| "logps/rejected": -0.9507579803466797, | |
| "loss": 0.7727, | |
| "odds_ratio_loss": 0.623470664024353, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.07103335857391357, | |
| "rewards/margins": 0.02404244802892208, | |
| "rewards/rejected": -0.09507580101490021, | |
| "sft_loss": 0.710333526134491, | |
| "step": 990 | |
| }, | |
| { | |
| "epoch": 1.616488179430188, | |
| "grad_norm": 1.4722503423690796, | |
| "learning_rate": 2.191736455761947e-06, | |
| "logits/chosen": -3.127290964126587, | |
| "logits/rejected": -3.1501638889312744, | |
| "logps/chosen": -0.6768070459365845, | |
| "logps/rejected": -0.8489478826522827, | |
| "loss": 0.7341, | |
| "odds_ratio_loss": 0.5731813311576843, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.06768069416284561, | |
| "rewards/margins": 0.017214089632034302, | |
| "rewards/rejected": -0.08489479124546051, | |
| "sft_loss": 0.6768070459365845, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 1.616488179430188, | |
| "eval_logits/chosen": -3.095952272415161, | |
| "eval_logits/rejected": -3.1193559169769287, | |
| "eval_logps/chosen": -0.7794692516326904, | |
| "eval_logps/rejected": -0.9804279208183289, | |
| "eval_loss": 0.8449718356132507, | |
| "eval_odds_ratio_loss": 0.655025839805603, | |
| "eval_rewards/accuracies": 0.5672727227210999, | |
| "eval_rewards/chosen": -0.07794692367315292, | |
| "eval_rewards/margins": 0.020095879212021828, | |
| "eval_rewards/rejected": -0.098042793571949, | |
| "eval_runtime": 367.195, | |
| "eval_samples_per_second": 2.996, | |
| "eval_sft_loss": 0.7794692516326904, | |
| "eval_steps_per_second": 1.498, | |
| "step": 1000 | |
| }, | |
| { | |
| "epoch": 1.6326530612244898, | |
| "grad_norm": 2.0673484802246094, | |
| "learning_rate": 2.1497413764574673e-06, | |
| "logits/chosen": -3.119420051574707, | |
| "logits/rejected": -3.1228458881378174, | |
| "logps/chosen": -0.7603039145469666, | |
| "logps/rejected": -1.017571210861206, | |
| "loss": 0.8184, | |
| "odds_ratio_loss": 0.5807704329490662, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.07603039592504501, | |
| "rewards/margins": 0.025726735591888428, | |
| "rewards/rejected": -0.10175712406635284, | |
| "sft_loss": 0.7603039145469666, | |
| "step": 1010 | |
| }, | |
| { | |
| "epoch": 1.6488179430187917, | |
| "grad_norm": 2.212749719619751, | |
| "learning_rate": 2.1078468757516395e-06, | |
| "logits/chosen": -3.0987765789031982, | |
| "logits/rejected": -3.1337482929229736, | |
| "logps/chosen": -0.7031766772270203, | |
| "logps/rejected": -0.9084986448287964, | |
| "loss": 0.762, | |
| "odds_ratio_loss": 0.5880716443061829, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.07031767070293427, | |
| "rewards/margins": 0.020532192662358284, | |
| "rewards/rejected": -0.0908498615026474, | |
| "sft_loss": 0.7031766772270203, | |
| "step": 1020 | |
| }, | |
| { | |
| "epoch": 1.6649828248130936, | |
| "grad_norm": 1.7040510177612305, | |
| "learning_rate": 2.0660649838698145e-06, | |
| "logits/chosen": -3.1251206398010254, | |
| "logits/rejected": -3.140993595123291, | |
| "logps/chosen": -0.7490108013153076, | |
| "logps/rejected": -0.9883764982223511, | |
| "loss": 0.8102, | |
| "odds_ratio_loss": 0.6117704510688782, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.07490108907222748, | |
| "rewards/margins": 0.023936569690704346, | |
| "rewards/rejected": -0.09883765131235123, | |
| "sft_loss": 0.7490108013153076, | |
| "step": 1030 | |
| }, | |
| { | |
| "epoch": 1.6811477066073954, | |
| "grad_norm": 2.5308990478515625, | |
| "learning_rate": 2.0244076987011284e-06, | |
| "logits/chosen": -3.156313419342041, | |
| "logits/rejected": -3.149254560470581, | |
| "logps/chosen": -0.7889419794082642, | |
| "logps/rejected": -0.9795150756835938, | |
| "loss": 0.8507, | |
| "odds_ratio_loss": 0.6177965998649597, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.07889419794082642, | |
| "rewards/margins": 0.019057301804423332, | |
| "rewards/rejected": -0.0979515090584755, | |
| "sft_loss": 0.7889419794082642, | |
| "step": 1040 | |
| }, | |
| { | |
| "epoch": 1.6973125884016973, | |
| "grad_norm": 1.713526725769043, | |
| "learning_rate": 1.982886982353251e-06, | |
| "logits/chosen": -3.1534323692321777, | |
| "logits/rejected": -3.143846035003662, | |
| "logps/chosen": -0.7670282125473022, | |
| "logps/rejected": -1.0469207763671875, | |
| "loss": 0.8306, | |
| "odds_ratio_loss": 0.635661244392395, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.07670283317565918, | |
| "rewards/margins": 0.027989249676465988, | |
| "rewards/rejected": -0.10469207912683487, | |
| "sft_loss": 0.7670282125473022, | |
| "step": 1050 | |
| }, | |
| { | |
| "epoch": 1.7134774701959992, | |
| "grad_norm": 2.0344197750091553, | |
| "learning_rate": 1.941514757717392e-06, | |
| "logits/chosen": -3.1048598289489746, | |
| "logits/rejected": -3.1271190643310547, | |
| "logps/chosen": -0.7640289068222046, | |
| "logps/rejected": -1.0232574939727783, | |
| "loss": 0.8194, | |
| "odds_ratio_loss": 0.5533130764961243, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.07640289515256882, | |
| "rewards/margins": 0.025922849774360657, | |
| "rewards/rejected": -0.10232573747634888, | |
| "sft_loss": 0.7640289068222046, | |
| "step": 1060 | |
| }, | |
| { | |
| "epoch": 1.729642351990301, | |
| "grad_norm": 4.370427131652832, | |
| "learning_rate": 1.9003029050445953e-06, | |
| "logits/chosen": -3.0902037620544434, | |
| "logits/rejected": -3.1260688304901123, | |
| "logps/chosen": -0.7717964053153992, | |
| "logps/rejected": -0.9319893717765808, | |
| "loss": 0.8345, | |
| "odds_ratio_loss": 0.6267774701118469, | |
| "rewards/accuracies": 0.5562499761581421, | |
| "rewards/chosen": -0.07717963308095932, | |
| "rewards/margins": 0.016019299626350403, | |
| "rewards/rejected": -0.09319894015789032, | |
| "sft_loss": 0.7717964053153992, | |
| "step": 1070 | |
| }, | |
| { | |
| "epoch": 1.745807233784603, | |
| "grad_norm": 2.206437110900879, | |
| "learning_rate": 1.8592632585342523e-06, | |
| "logits/chosen": -3.108377456665039, | |
| "logits/rejected": -3.137510299682617, | |
| "logps/chosen": -0.730410635471344, | |
| "logps/rejected": -0.9709598422050476, | |
| "loss": 0.7903, | |
| "odds_ratio_loss": 0.599312424659729, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.07304105907678604, | |
| "rewards/margins": 0.02405492775142193, | |
| "rewards/rejected": -0.09709598869085312, | |
| "sft_loss": 0.730410635471344, | |
| "step": 1080 | |
| }, | |
| { | |
| "epoch": 1.7619721155789048, | |
| "grad_norm": 3.54589581489563, | |
| "learning_rate": 1.8184076029358527e-06, | |
| "logits/chosen": -3.0873734951019287, | |
| "logits/rejected": -3.0916082859039307, | |
| "logps/chosen": -0.7298410534858704, | |
| "logps/rejected": -0.8651703000068665, | |
| "loss": 0.797, | |
| "odds_ratio_loss": 0.6711704134941101, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.07298411428928375, | |
| "rewards/margins": 0.01353292353451252, | |
| "rewards/rejected": -0.08651703596115112, | |
| "sft_loss": 0.7298410534858704, | |
| "step": 1090 | |
| }, | |
| { | |
| "epoch": 1.7781369973732066, | |
| "grad_norm": 1.9253672361373901, | |
| "learning_rate": 1.7777476701649318e-06, | |
| "logits/chosen": -3.0890607833862305, | |
| "logits/rejected": -3.1127114295959473, | |
| "logps/chosen": -0.7377376556396484, | |
| "logps/rejected": -0.9456714391708374, | |
| "loss": 0.7993, | |
| "odds_ratio_loss": 0.6157868504524231, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.07377376407384872, | |
| "rewards/margins": 0.02079339325428009, | |
| "rewards/rejected": -0.09456716477870941, | |
| "sft_loss": 0.7377376556396484, | |
| "step": 1100 | |
| }, | |
| { | |
| "epoch": 1.7943018791675085, | |
| "grad_norm": 2.634758949279785, | |
| "learning_rate": 1.7372951359341925e-06, | |
| "logits/chosen": -3.1294898986816406, | |
| "logits/rejected": -3.1275429725646973, | |
| "logps/chosen": -0.7182799577713013, | |
| "logps/rejected": -0.897103488445282, | |
| "loss": 0.7819, | |
| "odds_ratio_loss": 0.6362147331237793, | |
| "rewards/accuracies": 0.53125, | |
| "rewards/chosen": -0.07182799279689789, | |
| "rewards/margins": 0.01788235269486904, | |
| "rewards/rejected": -0.08971034735441208, | |
| "sft_loss": 0.7182799577713013, | |
| "step": 1110 | |
| }, | |
| { | |
| "epoch": 1.8104667609618104, | |
| "grad_norm": 4.11432409286499, | |
| "learning_rate": 1.6970616164007547e-06, | |
| "logits/chosen": -3.109255075454712, | |
| "logits/rejected": -3.1157066822052, | |
| "logps/chosen": -0.7095004320144653, | |
| "logps/rejected": -0.9157026410102844, | |
| "loss": 0.7723, | |
| "odds_ratio_loss": 0.6284032464027405, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.07095004618167877, | |
| "rewards/margins": 0.020620223134756088, | |
| "rewards/rejected": -0.09157026559114456, | |
| "sft_loss": 0.7095004320144653, | |
| "step": 1120 | |
| }, | |
| { | |
| "epoch": 1.8266316427561122, | |
| "grad_norm": 3.5484671592712402, | |
| "learning_rate": 1.6570586648305276e-06, | |
| "logits/chosen": -3.1144962310791016, | |
| "logits/rejected": -3.162351608276367, | |
| "logps/chosen": -0.7635061144828796, | |
| "logps/rejected": -0.9983874559402466, | |
| "loss": 0.8279, | |
| "odds_ratio_loss": 0.6435292959213257, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.0763506144285202, | |
| "rewards/margins": 0.023488130420446396, | |
| "rewards/rejected": -0.0998387485742569, | |
| "sft_loss": 0.7635061144828796, | |
| "step": 1130 | |
| }, | |
| { | |
| "epoch": 1.842796524550414, | |
| "grad_norm": 2.315484046936035, | |
| "learning_rate": 1.6172977682806151e-06, | |
| "logits/chosen": -3.1250674724578857, | |
| "logits/rejected": -3.169023036956787, | |
| "logps/chosen": -0.7526008486747742, | |
| "logps/rejected": -0.9915586709976196, | |
| "loss": 0.8118, | |
| "odds_ratio_loss": 0.5918548703193665, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.07526008039712906, | |
| "rewards/margins": 0.023895783349871635, | |
| "rewards/rejected": -0.09915586560964584, | |
| "sft_loss": 0.7526008486747742, | |
| "step": 1140 | |
| }, | |
| { | |
| "epoch": 1.858961406344716, | |
| "grad_norm": 1.8489255905151367, | |
| "learning_rate": 1.5777903443007586e-06, | |
| "logits/chosen": -3.0853919982910156, | |
| "logits/rejected": -3.067697286605835, | |
| "logps/chosen": -0.761858344078064, | |
| "logps/rejected": -1.018090844154358, | |
| "loss": 0.8238, | |
| "odds_ratio_loss": 0.6194515824317932, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.07618583738803864, | |
| "rewards/margins": 0.02562323771417141, | |
| "rewards/rejected": -0.1018090695142746, | |
| "sft_loss": 0.761858344078064, | |
| "step": 1150 | |
| }, | |
| { | |
| "epoch": 1.8751262881390178, | |
| "grad_norm": 3.776698589324951, | |
| "learning_rate": 1.5385477376547226e-06, | |
| "logits/chosen": -3.117295742034912, | |
| "logits/rejected": -3.1292173862457275, | |
| "logps/chosen": -0.7485045194625854, | |
| "logps/rejected": -0.9388057589530945, | |
| "loss": 0.8087, | |
| "odds_ratio_loss": 0.6023274064064026, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.07485045492649078, | |
| "rewards/margins": 0.019030118361115456, | |
| "rewards/rejected": -0.09388057142496109, | |
| "sft_loss": 0.7485045194625854, | |
| "step": 1160 | |
| }, | |
| { | |
| "epoch": 1.89129116993332, | |
| "grad_norm": 2.622969627380371, | |
| "learning_rate": 1.4995812170625845e-06, | |
| "logits/chosen": -3.100135564804077, | |
| "logits/rejected": -3.120415210723877, | |
| "logps/chosen": -0.7464675307273865, | |
| "logps/rejected": -1.0535166263580322, | |
| "loss": 0.8049, | |
| "odds_ratio_loss": 0.5842532515525818, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.07464675605297089, | |
| "rewards/margins": 0.030704926699399948, | |
| "rewards/rejected": -0.10535167157649994, | |
| "sft_loss": 0.7464675307273865, | |
| "step": 1170 | |
| }, | |
| { | |
| "epoch": 1.9074560517276218, | |
| "grad_norm": 2.950418710708618, | |
| "learning_rate": 1.4609019719648666e-06, | |
| "logits/chosen": -3.1377501487731934, | |
| "logits/rejected": -3.1489570140838623, | |
| "logps/chosen": -0.7555149793624878, | |
| "logps/rejected": -0.9784708023071289, | |
| "loss": 0.8149, | |
| "odds_ratio_loss": 0.5935994386672974, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.07555149495601654, | |
| "rewards/margins": 0.022295573726296425, | |
| "rewards/rejected": -0.09784707427024841, | |
| "sft_loss": 0.7555149793624878, | |
| "step": 1180 | |
| }, | |
| { | |
| "epoch": 1.9236209335219236, | |
| "grad_norm": 3.5587480068206787, | |
| "learning_rate": 1.42252110930943e-06, | |
| "logits/chosen": -3.077787399291992, | |
| "logits/rejected": -3.097930908203125, | |
| "logps/chosen": -0.6736657023429871, | |
| "logps/rejected": -0.9208394289016724, | |
| "loss": 0.7314, | |
| "odds_ratio_loss": 0.5776128768920898, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.0673665776848793, | |
| "rewards/margins": 0.024717364460229874, | |
| "rewards/rejected": -0.09208394587039948, | |
| "sft_loss": 0.6736657023429871, | |
| "step": 1190 | |
| }, | |
| { | |
| "epoch": 1.9397858153162255, | |
| "grad_norm": 4.154015064239502, | |
| "learning_rate": 1.3844496503620493e-06, | |
| "logits/chosen": -3.0993504524230957, | |
| "logits/rejected": -3.152251720428467, | |
| "logps/chosen": -0.781159520149231, | |
| "logps/rejected": -0.947624683380127, | |
| "loss": 0.8436, | |
| "odds_ratio_loss": 0.6244069933891296, | |
| "rewards/accuracies": 0.512499988079071, | |
| "rewards/chosen": -0.07811595499515533, | |
| "rewards/margins": 0.01664651185274124, | |
| "rewards/rejected": -0.09476246684789658, | |
| "sft_loss": 0.781159520149231, | |
| "step": 1200 | |
| }, | |
| { | |
| "epoch": 1.9559506971105274, | |
| "grad_norm": 3.2580513954162598, | |
| "learning_rate": 1.3466985275416081e-06, | |
| "logits/chosen": -3.074446439743042, | |
| "logits/rejected": -3.0826351642608643, | |
| "logps/chosen": -0.8331464529037476, | |
| "logps/rejected": -0.9720133543014526, | |
| "loss": 0.9006, | |
| "odds_ratio_loss": 0.6748316287994385, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.08331465721130371, | |
| "rewards/margins": 0.013886680826544762, | |
| "rewards/rejected": -0.09720133244991302, | |
| "sft_loss": 0.8331464529037476, | |
| "step": 1210 | |
| }, | |
| { | |
| "epoch": 1.9721155789048292, | |
| "grad_norm": 3.6283507347106934, | |
| "learning_rate": 1.309278581280791e-06, | |
| "logits/chosen": -3.1137681007385254, | |
| "logits/rejected": -3.1031088829040527, | |
| "logps/chosen": -0.6959558129310608, | |
| "logps/rejected": -0.9509153366088867, | |
| "loss": 0.7562, | |
| "odds_ratio_loss": 0.602826714515686, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.0695955827832222, | |
| "rewards/margins": 0.025495965033769608, | |
| "rewards/rejected": -0.09509153664112091, | |
| "sft_loss": 0.6959558129310608, | |
| "step": 1220 | |
| }, | |
| { | |
| "epoch": 1.9882804606991311, | |
| "grad_norm": 1.468146800994873, | |
| "learning_rate": 1.272200556913199e-06, | |
| "logits/chosen": -3.116267204284668, | |
| "logits/rejected": -3.1380016803741455, | |
| "logps/chosen": -0.7585796117782593, | |
| "logps/rejected": -0.9659263491630554, | |
| "loss": 0.827, | |
| "odds_ratio_loss": 0.6845245361328125, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.07585795968770981, | |
| "rewards/margins": 0.020734664052724838, | |
| "rewards/rejected": -0.09659262746572495, | |
| "sft_loss": 0.7585796117782593, | |
| "step": 1230 | |
| }, | |
| { | |
| "epoch": 2.004445342493433, | |
| "grad_norm": 2.9131782054901123, | |
| "learning_rate": 1.2354751015877698e-06, | |
| "logits/chosen": -3.0867080688476562, | |
| "logits/rejected": -3.120702028274536, | |
| "logps/chosen": -0.7115014791488647, | |
| "logps/rejected": -0.9811463356018066, | |
| "loss": 0.7717, | |
| "odds_ratio_loss": 0.6017391681671143, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.07115015387535095, | |
| "rewards/margins": 0.02696448564529419, | |
| "rewards/rejected": -0.09811463207006454, | |
| "sft_loss": 0.7115014791488647, | |
| "step": 1240 | |
| }, | |
| { | |
| "epoch": 2.020610224287735, | |
| "grad_norm": 5.614047527313232, | |
| "learning_rate": 1.1991127612113945e-06, | |
| "logits/chosen": -3.103969097137451, | |
| "logits/rejected": -3.1521923542022705, | |
| "logps/chosen": -0.748454749584198, | |
| "logps/rejected": -0.9711889028549194, | |
| "loss": 0.8062, | |
| "odds_ratio_loss": 0.5778619050979614, | |
| "rewards/accuracies": 0.59375, | |
| "rewards/chosen": -0.07484547793865204, | |
| "rewards/margins": 0.022273404523730278, | |
| "rewards/rejected": -0.09711887687444687, | |
| "sft_loss": 0.748454749584198, | |
| "step": 1250 | |
| }, | |
| { | |
| "epoch": 2.036775106082037, | |
| "grad_norm": 3.2708330154418945, | |
| "learning_rate": 1.1631239774206035e-06, | |
| "logits/chosen": -3.087298631668091, | |
| "logits/rejected": -3.0905699729919434, | |
| "logps/chosen": -0.7345898747444153, | |
| "logps/rejected": -0.9632331132888794, | |
| "loss": 0.7998, | |
| "odds_ratio_loss": 0.6524921655654907, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.07345898449420929, | |
| "rewards/margins": 0.022864321246743202, | |
| "rewards/rejected": -0.09632330387830734, | |
| "sft_loss": 0.7345898747444153, | |
| "step": 1260 | |
| }, | |
| { | |
| "epoch": 2.052939987876339, | |
| "grad_norm": 5.292441368103027, | |
| "learning_rate": 1.1275190845831978e-06, | |
| "logits/chosen": -3.094177007675171, | |
| "logits/rejected": -3.105118989944458, | |
| "logps/chosen": -0.6928293704986572, | |
| "logps/rejected": -0.9615745544433594, | |
| "loss": 0.7462, | |
| "odds_ratio_loss": 0.5332867503166199, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.06928294897079468, | |
| "rewards/margins": 0.026874512434005737, | |
| "rewards/rejected": -0.09615744650363922, | |
| "sft_loss": 0.6928293704986572, | |
| "step": 1270 | |
| }, | |
| { | |
| "epoch": 2.0691048696706407, | |
| "grad_norm": 2.8853020668029785, | |
| "learning_rate": 1.0923083068306778e-06, | |
| "logits/chosen": -3.1118180751800537, | |
| "logits/rejected": -3.0961227416992188, | |
| "logps/chosen": -0.739953339099884, | |
| "logps/rejected": -1.0457074642181396, | |
| "loss": 0.7958, | |
| "odds_ratio_loss": 0.5585684776306152, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.07399533689022064, | |
| "rewards/margins": 0.03057541325688362, | |
| "rewards/rejected": -0.10457074642181396, | |
| "sft_loss": 0.739953339099884, | |
| "step": 1280 | |
| }, | |
| { | |
| "epoch": 2.0852697514649425, | |
| "grad_norm": 2.046693801879883, | |
| "learning_rate": 1.0575017551223348e-06, | |
| "logits/chosen": -3.0827836990356445, | |
| "logits/rejected": -3.099609851837158, | |
| "logps/chosen": -0.6882905960083008, | |
| "logps/rejected": -0.9119550585746765, | |
| "loss": 0.7477, | |
| "odds_ratio_loss": 0.594234824180603, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.06882907450199127, | |
| "rewards/margins": 0.02236645109951496, | |
| "rewards/rejected": -0.09119551628828049, | |
| "sft_loss": 0.6882905960083008, | |
| "step": 1290 | |
| }, | |
| { | |
| "epoch": 2.1014346332592444, | |
| "grad_norm": 3.569880723953247, | |
| "learning_rate": 1.023109424341833e-06, | |
| "logits/chosen": -3.091522693634033, | |
| "logits/rejected": -3.1377675533294678, | |
| "logps/chosen": -0.7397576570510864, | |
| "logps/rejected": -0.9756298065185547, | |
| "loss": 0.8002, | |
| "odds_ratio_loss": 0.6045941114425659, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.07397577166557312, | |
| "rewards/margins": 0.023587211966514587, | |
| "rewards/rejected": -0.09756298363208771, | |
| "sft_loss": 0.7397576570510864, | |
| "step": 1300 | |
| }, | |
| { | |
| "epoch": 2.1175995150535463, | |
| "grad_norm": 2.92287278175354, | |
| "learning_rate": 9.891411904271273e-07, | |
| "logits/chosen": -3.091432571411133, | |
| "logits/rejected": -3.097942352294922, | |
| "logps/chosen": -0.7274054884910583, | |
| "logps/rejected": -0.9433088302612305, | |
| "loss": 0.7877, | |
| "odds_ratio_loss": 0.602813720703125, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.07274055480957031, | |
| "rewards/margins": 0.021590325981378555, | |
| "rewards/rejected": -0.09433087706565857, | |
| "sft_loss": 0.7274054884910583, | |
| "step": 1310 | |
| }, | |
| { | |
| "epoch": 2.133764396847848, | |
| "grad_norm": 1.6639999151229858, | |
| "learning_rate": 9.556068075345363e-07, | |
| "logits/chosen": -3.1349058151245117, | |
| "logits/rejected": -3.1192307472229004, | |
| "logps/chosen": -0.7446939945220947, | |
| "logps/rejected": -0.9326783418655396, | |
| "loss": 0.8047, | |
| "odds_ratio_loss": 0.5995923280715942, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.07446939498186111, | |
| "rewards/margins": 0.018798431381583214, | |
| "rewards/rejected": -0.09326782822608948, | |
| "sft_loss": 0.7446939945220947, | |
| "step": 1320 | |
| }, | |
| { | |
| "epoch": 2.14992927864215, | |
| "grad_norm": 1.6199049949645996, | |
| "learning_rate": 9.225159052377838e-07, | |
| "logits/chosen": -3.085836887359619, | |
| "logits/rejected": -3.1293978691101074, | |
| "logps/chosen": -0.7635077238082886, | |
| "logps/rejected": -1.0566623210906982, | |
| "loss": 0.8227, | |
| "odds_ratio_loss": 0.591996431350708, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.07635077089071274, | |
| "rewards/margins": 0.029315466061234474, | |
| "rewards/rejected": -0.10566625744104385, | |
| "sft_loss": 0.7635077238082886, | |
| "step": 1330 | |
| }, | |
| { | |
| "epoch": 2.166094160436452, | |
| "grad_norm": 14.56561279296875, | |
| "learning_rate": 8.898779857628184e-07, | |
| "logits/chosen": -3.0713887214660645, | |
| "logits/rejected": -3.125135898590088, | |
| "logps/chosen": -0.6671522259712219, | |
| "logps/rejected": -0.8694232702255249, | |
| "loss": 0.7265, | |
| "odds_ratio_loss": 0.5937641859054565, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.06671522557735443, | |
| "rewards/margins": 0.02022710070014, | |
| "rewards/rejected": -0.08694233000278473, | |
| "sft_loss": 0.6671522259712219, | |
| "step": 1340 | |
| }, | |
| { | |
| "epoch": 2.1822590422307537, | |
| "grad_norm": 2.1917502880096436, | |
| "learning_rate": 8.577024212591975e-07, | |
| "logits/chosen": -3.152409791946411, | |
| "logits/rejected": -3.142133951187134, | |
| "logps/chosen": -0.7823044657707214, | |
| "logps/rejected": -0.9282905459403992, | |
| "loss": 0.8458, | |
| "odds_ratio_loss": 0.6353241205215454, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.07823045551776886, | |
| "rewards/margins": 0.0145986033603549, | |
| "rewards/rejected": -0.09282905608415604, | |
| "sft_loss": 0.7823044657707214, | |
| "step": 1350 | |
| }, | |
| { | |
| "epoch": 2.1984239240250556, | |
| "grad_norm": 2.377164363861084, | |
| "learning_rate": 8.259984511088276e-07, | |
| "logits/chosen": -3.0746350288391113, | |
| "logits/rejected": -3.107931613922119, | |
| "logps/chosen": -0.7670485973358154, | |
| "logps/rejected": -0.9791353344917297, | |
| "loss": 0.8321, | |
| "odds_ratio_loss": 0.6501604914665222, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.07670485973358154, | |
| "rewards/margins": 0.02120867185294628, | |
| "rewards/rejected": -0.09791352599859238, | |
| "sft_loss": 0.7670485973358154, | |
| "step": 1360 | |
| }, | |
| { | |
| "epoch": 2.2145888058193575, | |
| "grad_norm": 1.4493186473846436, | |
| "learning_rate": 7.947751792728237e-07, | |
| "logits/chosen": -3.1149420738220215, | |
| "logits/rejected": -3.106590986251831, | |
| "logps/chosen": -0.736428439617157, | |
| "logps/rejected": -1.0205104351043701, | |
| "loss": 0.7973, | |
| "odds_ratio_loss": 0.6086055040359497, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.07364284247159958, | |
| "rewards/margins": 0.02840820886194706, | |
| "rewards/rejected": -0.10205105692148209, | |
| "sft_loss": 0.736428439617157, | |
| "step": 1370 | |
| }, | |
| { | |
| "epoch": 2.2307536876136593, | |
| "grad_norm": 2.596027374267578, | |
| "learning_rate": 7.640415716772626e-07, | |
| "logits/chosen": -3.1113359928131104, | |
| "logits/rejected": -3.1498453617095947, | |
| "logps/chosen": -0.7384335398674011, | |
| "logps/rejected": -1.0052040815353394, | |
| "loss": 0.7991, | |
| "odds_ratio_loss": 0.606296718120575, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.07384335994720459, | |
| "rewards/margins": 0.026677042245864868, | |
| "rewards/rejected": -0.10052040964365005, | |
| "sft_loss": 0.7384335398674011, | |
| "step": 1380 | |
| }, | |
| { | |
| "epoch": 2.246918569407961, | |
| "grad_norm": 1.8183890581130981, | |
| "learning_rate": 7.338064536385722e-07, | |
| "logits/chosen": -3.1014134883880615, | |
| "logits/rejected": -3.1173253059387207, | |
| "logps/chosen": -0.7216871380805969, | |
| "logps/rejected": -1.0171915292739868, | |
| "loss": 0.7805, | |
| "odds_ratio_loss": 0.5884080529212952, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.07216870784759521, | |
| "rewards/margins": 0.029550448060035706, | |
| "rewards/rejected": -0.10171915590763092, | |
| "sft_loss": 0.7216871380805969, | |
| "step": 1390 | |
| }, | |
| { | |
| "epoch": 2.263083451202263, | |
| "grad_norm": 2.7193105220794678, | |
| "learning_rate": 7.040785073292883e-07, | |
| "logits/chosen": -3.052278995513916, | |
| "logits/rejected": -3.0792136192321777, | |
| "logps/chosen": -0.8029114603996277, | |
| "logps/rejected": -1.0175930261611938, | |
| "loss": 0.8702, | |
| "odds_ratio_loss": 0.6730188727378845, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.08029113709926605, | |
| "rewards/margins": 0.02146816998720169, | |
| "rewards/rejected": -0.10175931453704834, | |
| "sft_loss": 0.8029114603996277, | |
| "step": 1400 | |
| }, | |
| { | |
| "epoch": 2.279248332996565, | |
| "grad_norm": 1.6374540328979492, | |
| "learning_rate": 6.748662692849297e-07, | |
| "logits/chosen": -3.0864005088806152, | |
| "logits/rejected": -3.1004090309143066, | |
| "logps/chosen": -0.6923743486404419, | |
| "logps/rejected": -1.0528560876846313, | |
| "loss": 0.747, | |
| "odds_ratio_loss": 0.5459417104721069, | |
| "rewards/accuracies": 0.6499999761581421, | |
| "rewards/chosen": -0.06923744082450867, | |
| "rewards/margins": 0.03604819253087044, | |
| "rewards/rejected": -0.10528562217950821, | |
| "sft_loss": 0.6923743486404419, | |
| "step": 1410 | |
| }, | |
| { | |
| "epoch": 2.295413214790867, | |
| "grad_norm": 5.552002429962158, | |
| "learning_rate": 6.46178127952686e-07, | |
| "logits/chosen": -3.1168243885040283, | |
| "logits/rejected": -3.132902145385742, | |
| "logps/chosen": -0.7120259404182434, | |
| "logps/rejected": -0.9511427879333496, | |
| "loss": 0.7669, | |
| "odds_ratio_loss": 0.5487939119338989, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.0712025836110115, | |
| "rewards/margins": 0.023911695927381516, | |
| "rewards/rejected": -0.09511429071426392, | |
| "sft_loss": 0.7120259404182434, | |
| "step": 1420 | |
| }, | |
| { | |
| "epoch": 2.3115780965851687, | |
| "grad_norm": 6.720409870147705, | |
| "learning_rate": 6.180223212826289e-07, | |
| "logits/chosen": -3.1055123805999756, | |
| "logits/rejected": -3.0998032093048096, | |
| "logps/chosen": -0.7326000928878784, | |
| "logps/rejected": -0.947357177734375, | |
| "loss": 0.7933, | |
| "odds_ratio_loss": 0.6070013046264648, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.07326001673936844, | |
| "rewards/margins": 0.021475713700056076, | |
| "rewards/rejected": -0.09473572671413422, | |
| "sft_loss": 0.7326000928878784, | |
| "step": 1430 | |
| }, | |
| { | |
| "epoch": 2.3277429783794705, | |
| "grad_norm": 1.7139315605163574, | |
| "learning_rate": 5.904069343621443e-07, | |
| "logits/chosen": -3.117607593536377, | |
| "logits/rejected": -3.1124439239501953, | |
| "logps/chosen": -0.7263907194137573, | |
| "logps/rejected": -0.9684870839118958, | |
| "loss": 0.7844, | |
| "odds_ratio_loss": 0.5803811550140381, | |
| "rewards/accuracies": 0.6312500238418579, | |
| "rewards/chosen": -0.07263907045125961, | |
| "rewards/margins": 0.024209631606936455, | |
| "rewards/rejected": -0.09684871137142181, | |
| "sft_loss": 0.7263907194137573, | |
| "step": 1440 | |
| }, | |
| { | |
| "epoch": 2.3439078601737724, | |
| "grad_norm": 4.18460750579834, | |
| "learning_rate": 5.633398970942544e-07, | |
| "logits/chosen": -3.0699353218078613, | |
| "logits/rejected": -3.1113662719726562, | |
| "logps/chosen": -0.71888267993927, | |
| "logps/rejected": -0.9213592410087585, | |
| "loss": 0.7809, | |
| "odds_ratio_loss": 0.6200512647628784, | |
| "rewards/accuracies": 0.5874999761581421, | |
| "rewards/chosen": -0.0718882754445076, | |
| "rewards/margins": 0.020247649401426315, | |
| "rewards/rejected": -0.09213592857122421, | |
| "sft_loss": 0.71888267993927, | |
| "step": 1450 | |
| }, | |
| { | |
| "epoch": 2.3600727419680743, | |
| "grad_norm": 2.875581741333008, | |
| "learning_rate": 5.368289819205069e-07, | |
| "logits/chosen": -3.1074984073638916, | |
| "logits/rejected": -3.116929054260254, | |
| "logps/chosen": -0.675830602645874, | |
| "logps/rejected": -0.9225249290466309, | |
| "loss": 0.7373, | |
| "odds_ratio_loss": 0.6144708395004272, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.06758306175470352, | |
| "rewards/margins": 0.02466944232583046, | |
| "rewards/rejected": -0.09225250035524368, | |
| "sft_loss": 0.675830602645874, | |
| "step": 1460 | |
| }, | |
| { | |
| "epoch": 2.376237623762376, | |
| "grad_norm": 3.366608142852783, | |
| "learning_rate": 5.108818015890785e-07, | |
| "logits/chosen": -3.1295876502990723, | |
| "logits/rejected": -3.1546216011047363, | |
| "logps/chosen": -0.8030446171760559, | |
| "logps/rejected": -0.9700073003768921, | |
| "loss": 0.8661, | |
| "odds_ratio_loss": 0.6306995153427124, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.08030445873737335, | |
| "rewards/margins": 0.016696274280548096, | |
| "rewards/rejected": -0.09700073301792145, | |
| "sft_loss": 0.8030446171760559, | |
| "step": 1470 | |
| }, | |
| { | |
| "epoch": 2.392402505556678, | |
| "grad_norm": 2.7780675888061523, | |
| "learning_rate": 4.855058069687291e-07, | |
| "logits/chosen": -3.067026376724243, | |
| "logits/rejected": -3.0960917472839355, | |
| "logps/chosen": -0.7043627500534058, | |
| "logps/rejected": -0.9909790754318237, | |
| "loss": 0.7594, | |
| "odds_ratio_loss": 0.5507315397262573, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.07043628394603729, | |
| "rewards/margins": 0.028661629185080528, | |
| "rewards/rejected": -0.09909790754318237, | |
| "sft_loss": 0.7043627500534058, | |
| "step": 1480 | |
| }, | |
| { | |
| "epoch": 2.40856738735098, | |
| "grad_norm": 3.2751307487487793, | |
| "learning_rate": 4.607082849092523e-07, | |
| "logits/chosen": -3.099853277206421, | |
| "logits/rejected": -3.1020543575286865, | |
| "logps/chosen": -0.7886861562728882, | |
| "logps/rejected": -0.976923942565918, | |
| "loss": 0.8511, | |
| "odds_ratio_loss": 0.6243327856063843, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.07886861264705658, | |
| "rewards/margins": 0.018823768943548203, | |
| "rewards/rejected": -0.09769239276647568, | |
| "sft_loss": 0.7886861562728882, | |
| "step": 1490 | |
| }, | |
| { | |
| "epoch": 2.4247322691452817, | |
| "grad_norm": 3.08627986907959, | |
| "learning_rate": 4.3649635614901405e-07, | |
| "logits/chosen": -3.0676140785217285, | |
| "logits/rejected": -3.144824504852295, | |
| "logps/chosen": -0.704692006111145, | |
| "logps/rejected": -0.8520969152450562, | |
| "loss": 0.7691, | |
| "odds_ratio_loss": 0.6437775492668152, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.0704691931605339, | |
| "rewards/margins": 0.014740494079887867, | |
| "rewards/rejected": -0.0852096900343895, | |
| "sft_loss": 0.704692006111145, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 2.4247322691452817, | |
| "eval_logits/chosen": -3.083430051803589, | |
| "eval_logits/rejected": -3.107269287109375, | |
| "eval_logps/chosen": -0.777300238609314, | |
| "eval_logps/rejected": -0.9876723289489746, | |
| "eval_loss": 0.8434417247772217, | |
| "eval_odds_ratio_loss": 0.6614136099815369, | |
| "eval_rewards/accuracies": 0.5690909028053284, | |
| "eval_rewards/chosen": -0.07773003727197647, | |
| "eval_rewards/margins": 0.021037202328443527, | |
| "eval_rewards/rejected": -0.0987672358751297, | |
| "eval_runtime": 371.5479, | |
| "eval_samples_per_second": 2.961, | |
| "eval_sft_loss": 0.777300238609314, | |
| "eval_steps_per_second": 1.48, | |
| "step": 1500 | |
| }, | |
| { | |
| "epoch": 2.4408971509395836, | |
| "grad_norm": 2.1558029651641846, | |
| "learning_rate": 4.128769732701973e-07, | |
| "logits/chosen": -3.0588672161102295, | |
| "logits/rejected": -3.053375720977783, | |
| "logps/chosen": -0.7354205250740051, | |
| "logps/rejected": -0.9381022453308105, | |
| "loss": 0.7959, | |
| "odds_ratio_loss": 0.6045327186584473, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.0735420510172844, | |
| "rewards/margins": 0.02026817761361599, | |
| "rewards/rejected": -0.09381023049354553, | |
| "sft_loss": 0.7354205250740051, | |
| "step": 1510 | |
| }, | |
| { | |
| "epoch": 2.4570620327338855, | |
| "grad_norm": 3.434962749481201, | |
| "learning_rate": 3.8985691870233046e-07, | |
| "logits/chosen": -3.1096482276916504, | |
| "logits/rejected": -3.1095433235168457, | |
| "logps/chosen": -0.7352281212806702, | |
| "logps/rejected": -0.9943790435791016, | |
| "loss": 0.7949, | |
| "odds_ratio_loss": 0.596770703792572, | |
| "rewards/accuracies": 0.543749988079071, | |
| "rewards/chosen": -0.07352282106876373, | |
| "rewards/margins": 0.025915095582604408, | |
| "rewards/rejected": -0.0994379073381424, | |
| "sft_loss": 0.7352281212806702, | |
| "step": 1520 | |
| }, | |
| { | |
| "epoch": 2.4732269145281873, | |
| "grad_norm": 2.586085557937622, | |
| "learning_rate": 3.6744280277467904e-07, | |
| "logits/chosen": -3.1208150386810303, | |
| "logits/rejected": -3.1456665992736816, | |
| "logps/chosen": -0.7365155816078186, | |
| "logps/rejected": -0.9590933918952942, | |
| "loss": 0.7993, | |
| "odds_ratio_loss": 0.6281741857528687, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.07365155220031738, | |
| "rewards/margins": 0.022257793694734573, | |
| "rewards/rejected": -0.09590934216976166, | |
| "sft_loss": 0.7365155816078186, | |
| "step": 1530 | |
| }, | |
| { | |
| "epoch": 2.489391796322489, | |
| "grad_norm": 6.254690170288086, | |
| "learning_rate": 3.456410618180503e-07, | |
| "logits/chosen": -3.0379281044006348, | |
| "logits/rejected": -3.0891363620758057, | |
| "logps/chosen": -0.6726978421211243, | |
| "logps/rejected": -1.0303423404693604, | |
| "loss": 0.7295, | |
| "odds_ratio_loss": 0.5683592557907104, | |
| "rewards/accuracies": 0.6625000238418579, | |
| "rewards/chosen": -0.06726977974176407, | |
| "rewards/margins": 0.03576444461941719, | |
| "rewards/rejected": -0.10303423553705215, | |
| "sft_loss": 0.6726978421211243, | |
| "step": 1540 | |
| }, | |
| { | |
| "epoch": 2.5055566781167915, | |
| "grad_norm": 2.056490898132324, | |
| "learning_rate": 3.244579563165753e-07, | |
| "logits/chosen": -3.0974183082580566, | |
| "logits/rejected": -3.1228187084198, | |
| "logps/chosen": -0.711591362953186, | |
| "logps/rejected": -1.0496152639389038, | |
| "loss": 0.7689, | |
| "odds_ratio_loss": 0.5726686716079712, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.07115913927555084, | |
| "rewards/margins": 0.033802393823862076, | |
| "rewards/rejected": -0.10496153682470322, | |
| "sft_loss": 0.711591362953186, | |
| "step": 1550 | |
| }, | |
| { | |
| "epoch": 2.521721559911093, | |
| "grad_norm": 1.438101053237915, | |
| "learning_rate": 3.038995691099697e-07, | |
| "logits/chosen": -3.08099365234375, | |
| "logits/rejected": -3.1116271018981934, | |
| "logps/chosen": -0.7405564188957214, | |
| "logps/rejected": -1.075670838356018, | |
| "loss": 0.7985, | |
| "odds_ratio_loss": 0.5797412991523743, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.07405563443899155, | |
| "rewards/margins": 0.0335114449262619, | |
| "rewards/rejected": -0.10756708681583405, | |
| "sft_loss": 0.7405564188957214, | |
| "step": 1560 | |
| }, | |
| { | |
| "epoch": 2.5378864417053952, | |
| "grad_norm": 9.284449577331543, | |
| "learning_rate": 2.839718036468192e-07, | |
| "logits/chosen": -3.127392530441284, | |
| "logits/rejected": -3.1654505729675293, | |
| "logps/chosen": -0.841861367225647, | |
| "logps/rejected": -0.9995994567871094, | |
| "loss": 0.9096, | |
| "odds_ratio_loss": 0.6777136921882629, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.0841861441731453, | |
| "rewards/margins": 0.015773817896842957, | |
| "rewards/rejected": -0.09995995461940765, | |
| "sft_loss": 0.841861367225647, | |
| "step": 1570 | |
| }, | |
| { | |
| "epoch": 2.5540513234996967, | |
| "grad_norm": 2.9508066177368164, | |
| "learning_rate": 2.646803822893723e-07, | |
| "logits/chosen": -3.1416640281677246, | |
| "logits/rejected": -3.1590213775634766, | |
| "logps/chosen": -0.7539080381393433, | |
| "logps/rejected": -0.9754577875137329, | |
| "loss": 0.8158, | |
| "odds_ratio_loss": 0.618767499923706, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.07539081573486328, | |
| "rewards/margins": 0.02215495891869068, | |
| "rewards/rejected": -0.09754578024148941, | |
| "sft_loss": 0.7539080381393433, | |
| "step": 1580 | |
| }, | |
| { | |
| "epoch": 2.570216205293999, | |
| "grad_norm": 2.8406593799591064, | |
| "learning_rate": 2.460308446703341e-07, | |
| "logits/chosen": -3.1490259170532227, | |
| "logits/rejected": -3.129563808441162, | |
| "logps/chosen": -0.7597383260726929, | |
| "logps/rejected": -0.9107489585876465, | |
| "loss": 0.8268, | |
| "odds_ratio_loss": 0.671094536781311, | |
| "rewards/accuracies": 0.5625, | |
| "rewards/chosen": -0.07597382366657257, | |
| "rewards/margins": 0.01510106772184372, | |
| "rewards/rejected": -0.09107489883899689, | |
| "sft_loss": 0.7597383260726929, | |
| "step": 1590 | |
| }, | |
| { | |
| "epoch": 2.5863810870883004, | |
| "grad_norm": 3.932293653488159, | |
| "learning_rate": 2.2802854610213143e-07, | |
| "logits/chosen": -3.0843563079833984, | |
| "logits/rejected": -3.080658197402954, | |
| "logps/chosen": -0.6698503494262695, | |
| "logps/rejected": -1.0470666885375977, | |
| "loss": 0.7238, | |
| "odds_ratio_loss": 0.5398669838905334, | |
| "rewards/accuracies": 0.668749988079071, | |
| "rewards/chosen": -0.06698504090309143, | |
| "rewards/margins": 0.03772163391113281, | |
| "rewards/rejected": -0.10470668226480484, | |
| "sft_loss": 0.6698503494262695, | |
| "step": 1600 | |
| }, | |
| { | |
| "epoch": 2.6025459688826027, | |
| "grad_norm": 3.3689260482788086, | |
| "learning_rate": 2.106786560391072e-07, | |
| "logits/chosen": -3.08052134513855, | |
| "logits/rejected": -3.1363070011138916, | |
| "logps/chosen": -0.7746056914329529, | |
| "logps/rejected": -0.9704595804214478, | |
| "loss": 0.8356, | |
| "odds_ratio_loss": 0.6099545359611511, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.07746056467294693, | |
| "rewards/margins": 0.019585389643907547, | |
| "rewards/rejected": -0.09704595804214478, | |
| "sft_loss": 0.7746056914329529, | |
| "step": 1610 | |
| }, | |
| { | |
| "epoch": 2.6187108506769046, | |
| "grad_norm": 1.7709842920303345, | |
| "learning_rate": 1.9398615659308255e-07, | |
| "logits/chosen": -3.1098244190216064, | |
| "logits/rejected": -3.1587026119232178, | |
| "logps/chosen": -0.7326648831367493, | |
| "logps/rejected": -0.9062278866767883, | |
| "loss": 0.7951, | |
| "odds_ratio_loss": 0.624650776386261, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.07326649129390717, | |
| "rewards/margins": 0.017356308177113533, | |
| "rewards/rejected": -0.09062279760837555, | |
| "sft_loss": 0.7326648831367493, | |
| "step": 1620 | |
| }, | |
| { | |
| "epoch": 2.6348757324712064, | |
| "grad_norm": 2.0778872966766357, | |
| "learning_rate": 1.7795584110272184e-07, | |
| "logits/chosen": -3.1534359455108643, | |
| "logits/rejected": -3.1423211097717285, | |
| "logps/chosen": -0.7454973459243774, | |
| "logps/rejected": -0.936767578125, | |
| "loss": 0.8102, | |
| "odds_ratio_loss": 0.6473931670188904, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.07454973459243774, | |
| "rewards/margins": 0.019127020612359047, | |
| "rewards/rejected": -0.09367676079273224, | |
| "sft_loss": 0.7454973459243774, | |
| "step": 1630 | |
| }, | |
| { | |
| "epoch": 2.6510406142655083, | |
| "grad_norm": 3.814206838607788, | |
| "learning_rate": 1.6259231275709636e-07, | |
| "logits/chosen": -3.149001121520996, | |
| "logits/rejected": -3.1615355014801025, | |
| "logps/chosen": -0.716022789478302, | |
| "logps/rejected": -0.8857117891311646, | |
| "loss": 0.7825, | |
| "odds_ratio_loss": 0.6644307971000671, | |
| "rewards/accuracies": 0.5687500238418579, | |
| "rewards/chosen": -0.07160228490829468, | |
| "rewards/margins": 0.016968905925750732, | |
| "rewards/rejected": -0.08857118338346481, | |
| "sft_loss": 0.716022789478302, | |
| "step": 1640 | |
| }, | |
| { | |
| "epoch": 2.66720549605981, | |
| "grad_norm": 1.4002131223678589, | |
| "learning_rate": 1.478999832738548e-07, | |
| "logits/chosen": -3.1335608959198, | |
| "logits/rejected": -3.1366758346557617, | |
| "logps/chosen": -0.7220789194107056, | |
| "logps/rejected": -1.0130765438079834, | |
| "loss": 0.7818, | |
| "odds_ratio_loss": 0.597649335861206, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.07220789790153503, | |
| "rewards/margins": 0.02909977361559868, | |
| "rewards/rejected": -0.10130767524242401, | |
| "sft_loss": 0.7220789194107056, | |
| "step": 1650 | |
| }, | |
| { | |
| "epoch": 2.683370377854112, | |
| "grad_norm": 2.5022165775299072, | |
| "learning_rate": 1.338830716323769e-07, | |
| "logits/chosen": -3.0961499214172363, | |
| "logits/rejected": -3.1152243614196777, | |
| "logps/chosen": -0.7230523228645325, | |
| "logps/rejected": -0.8930460810661316, | |
| "loss": 0.7852, | |
| "odds_ratio_loss": 0.6216701865196228, | |
| "rewards/accuracies": 0.581250011920929, | |
| "rewards/chosen": -0.07230523973703384, | |
| "rewards/margins": 0.016999371349811554, | |
| "rewards/rejected": -0.0893046110868454, | |
| "sft_loss": 0.7230523228645325, | |
| "step": 1660 | |
| }, | |
| { | |
| "epoch": 2.699535259648414, | |
| "grad_norm": 3.8939476013183594, | |
| "learning_rate": 1.205456028622723e-07, | |
| "logits/chosen": -3.1416923999786377, | |
| "logits/rejected": -3.130157232284546, | |
| "logps/chosen": -0.6890848875045776, | |
| "logps/rejected": -0.9714914560317993, | |
| "loss": 0.7441, | |
| "odds_ratio_loss": 0.550174355506897, | |
| "rewards/accuracies": 0.6187499761581421, | |
| "rewards/chosen": -0.06890848278999329, | |
| "rewards/margins": 0.028240656480193138, | |
| "rewards/rejected": -0.09714914858341217, | |
| "sft_loss": 0.6890848875045776, | |
| "step": 1670 | |
| }, | |
| { | |
| "epoch": 2.7157001414427158, | |
| "grad_norm": 1.6698890924453735, | |
| "learning_rate": 1.0789140688756805e-07, | |
| "logits/chosen": -3.1566967964172363, | |
| "logits/rejected": -3.149425745010376, | |
| "logps/chosen": -0.7115526795387268, | |
| "logps/rejected": -0.9621270298957825, | |
| "loss": 0.7691, | |
| "odds_ratio_loss": 0.5751715898513794, | |
| "rewards/accuracies": 0.625, | |
| "rewards/chosen": -0.07115526497364044, | |
| "rewards/margins": 0.025057435035705566, | |
| "rewards/rejected": -0.09621270000934601, | |
| "sft_loss": 0.7115526795387268, | |
| "step": 1680 | |
| }, | |
| { | |
| "epoch": 2.7318650232370176, | |
| "grad_norm": 5.264458179473877, | |
| "learning_rate": 9.592411742693098e-08, | |
| "logits/chosen": -3.078084945678711, | |
| "logits/rejected": -3.0921549797058105, | |
| "logps/chosen": -0.7386698722839355, | |
| "logps/rejected": -0.9247487783432007, | |
| "loss": 0.8064, | |
| "odds_ratio_loss": 0.6777487397193909, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.07386698573827744, | |
| "rewards/margins": 0.018607888370752335, | |
| "rewards/rejected": -0.09247487038373947, | |
| "sft_loss": 0.7386698722839355, | |
| "step": 1690 | |
| }, | |
| { | |
| "epoch": 2.7480299050313195, | |
| "grad_norm": 2.103006362915039, | |
| "learning_rate": 8.464717095022168e-08, | |
| "logits/chosen": -3.064146041870117, | |
| "logits/rejected": -3.0713071823120117, | |
| "logps/chosen": -0.7025493383407593, | |
| "logps/rejected": -0.955985426902771, | |
| "loss": 0.7608, | |
| "odds_ratio_loss": 0.5827249884605408, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.07025493681430817, | |
| "rewards/margins": 0.025343608111143112, | |
| "rewards/rejected": -0.09559854120016098, | |
| "sft_loss": 0.7025493383407593, | |
| "step": 1700 | |
| }, | |
| { | |
| "epoch": 2.7641947868256214, | |
| "grad_norm": 2.800687789916992, | |
| "learning_rate": 7.406380569169841e-08, | |
| "logits/chosen": -3.0971622467041016, | |
| "logits/rejected": -3.142702341079712, | |
| "logps/chosen": -0.752859890460968, | |
| "logps/rejected": -0.8985753059387207, | |
| "loss": 0.8177, | |
| "odds_ratio_loss": 0.6488373875617981, | |
| "rewards/accuracies": 0.518750011920929, | |
| "rewards/chosen": -0.07528599351644516, | |
| "rewards/margins": 0.014571529813110828, | |
| "rewards/rejected": -0.08985751867294312, | |
| "sft_loss": 0.752859890460968, | |
| "step": 1710 | |
| }, | |
| { | |
| "epoch": 2.7803596686199232, | |
| "grad_norm": 11.731331825256348, | |
| "learning_rate": 6.417706072013808e-08, | |
| "logits/chosen": -3.1208655834198, | |
| "logits/rejected": -3.1548678874969482, | |
| "logps/chosen": -0.7304657101631165, | |
| "logps/rejected": -0.9190757870674133, | |
| "loss": 0.7931, | |
| "odds_ratio_loss": 0.6268162727355957, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.07304656505584717, | |
| "rewards/margins": 0.01886100508272648, | |
| "rewards/rejected": -0.0919075757265091, | |
| "sft_loss": 0.7304657101631165, | |
| "step": 1720 | |
| }, | |
| { | |
| "epoch": 2.796524550414225, | |
| "grad_norm": 4.022006034851074, | |
| "learning_rate": 5.498977506615294e-08, | |
| "logits/chosen": -3.1070895195007324, | |
| "logits/rejected": -3.145142078399658, | |
| "logps/chosen": -0.7585607767105103, | |
| "logps/rejected": -0.914089560508728, | |
| "loss": 0.8244, | |
| "odds_ratio_loss": 0.6582176089286804, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.07585608214139938, | |
| "rewards/margins": 0.015552881173789501, | |
| "rewards/rejected": -0.09140896052122116, | |
| "sft_loss": 0.7585607767105103, | |
| "step": 1730 | |
| }, | |
| { | |
| "epoch": 2.812689432208527, | |
| "grad_norm": 2.081886053085327, | |
| "learning_rate": 4.6504586906947756e-08, | |
| "logits/chosen": -3.1400046348571777, | |
| "logits/rejected": -3.159885883331299, | |
| "logps/chosen": -0.7848642468452454, | |
| "logps/rejected": -0.9642190933227539, | |
| "loss": 0.8438, | |
| "odds_ratio_loss": 0.5897284746170044, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.07848642766475677, | |
| "rewards/margins": 0.01793549209833145, | |
| "rewards/rejected": -0.09642191231250763, | |
| "sft_loss": 0.7848642468452454, | |
| "step": 1740 | |
| }, | |
| { | |
| "epoch": 2.828854314002829, | |
| "grad_norm": 9.670836448669434, | |
| "learning_rate": 3.8723932808754914e-08, | |
| "logits/chosen": -3.1591954231262207, | |
| "logits/rejected": -3.179676055908203, | |
| "logps/chosen": -0.832345187664032, | |
| "logps/rejected": -0.9517787098884583, | |
| "loss": 0.8997, | |
| "odds_ratio_loss": 0.6738894581794739, | |
| "rewards/accuracies": 0.5375000238418579, | |
| "rewards/chosen": -0.0832345113158226, | |
| "rewards/margins": 0.01194334588944912, | |
| "rewards/rejected": -0.09517785906791687, | |
| "sft_loss": 0.832345187664032, | |
| "step": 1750 | |
| }, | |
| { | |
| "epoch": 2.8450191957971307, | |
| "grad_norm": 4.27302360534668, | |
| "learning_rate": 3.1650047027158014e-08, | |
| "logits/chosen": -3.132617473602295, | |
| "logits/rejected": -3.1612956523895264, | |
| "logps/chosen": -0.7341474294662476, | |
| "logps/rejected": -0.9550244212150574, | |
| "loss": 0.7929, | |
| "odds_ratio_loss": 0.5877509117126465, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.07341475039720535, | |
| "rewards/margins": 0.0220877043902874, | |
| "rewards/rejected": -0.09550245106220245, | |
| "sft_loss": 0.7341474294662476, | |
| "step": 1760 | |
| }, | |
| { | |
| "epoch": 2.8611840775914326, | |
| "grad_norm": 3.392895460128784, | |
| "learning_rate": 2.5284960865517848e-08, | |
| "logits/chosen": -3.093036413192749, | |
| "logits/rejected": -3.1320202350616455, | |
| "logps/chosen": -0.6883664727210999, | |
| "logps/rejected": -0.9603809118270874, | |
| "loss": 0.7445, | |
| "odds_ratio_loss": 0.5616418123245239, | |
| "rewards/accuracies": 0.606249988079071, | |
| "rewards/chosen": -0.06883664429187775, | |
| "rewards/margins": 0.027201462537050247, | |
| "rewards/rejected": -0.0960381031036377, | |
| "sft_loss": 0.6883664727210999, | |
| "step": 1770 | |
| }, | |
| { | |
| "epoch": 2.8773489593857344, | |
| "grad_norm": 2.937486410140991, | |
| "learning_rate": 1.9630502091670388e-08, | |
| "logits/chosen": -3.09773325920105, | |
| "logits/rejected": -3.1201109886169434, | |
| "logps/chosen": -0.7190070152282715, | |
| "logps/rejected": -0.9666651487350464, | |
| "loss": 0.7761, | |
| "odds_ratio_loss": 0.5710408687591553, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.07190070301294327, | |
| "rewards/margins": 0.02476580999791622, | |
| "rewards/rejected": -0.09666652232408524, | |
| "sft_loss": 0.7190070152282715, | |
| "step": 1780 | |
| }, | |
| { | |
| "epoch": 2.8935138411800363, | |
| "grad_norm": 3.6368203163146973, | |
| "learning_rate": 1.4688294413074677e-08, | |
| "logits/chosen": -3.076406717300415, | |
| "logits/rejected": -3.1111807823181152, | |
| "logps/chosen": -0.6571230888366699, | |
| "logps/rejected": -0.9525207281112671, | |
| "loss": 0.7146, | |
| "odds_ratio_loss": 0.5746604204177856, | |
| "rewards/accuracies": 0.6000000238418579, | |
| "rewards/chosen": -0.06571231037378311, | |
| "rewards/margins": 0.02953975833952427, | |
| "rewards/rejected": -0.09525207430124283, | |
| "sft_loss": 0.6571230888366699, | |
| "step": 1790 | |
| }, | |
| { | |
| "epoch": 2.909678722974338, | |
| "grad_norm": 2.2510814666748047, | |
| "learning_rate": 1.0459757010556626e-08, | |
| "logits/chosen": -3.1042840480804443, | |
| "logits/rejected": -3.1341700553894043, | |
| "logps/chosen": -0.7357559204101562, | |
| "logps/rejected": -0.8860123753547668, | |
| "loss": 0.7989, | |
| "odds_ratio_loss": 0.6314257383346558, | |
| "rewards/accuracies": 0.550000011920929, | |
| "rewards/chosen": -0.07357560098171234, | |
| "rewards/margins": 0.015025639906525612, | |
| "rewards/rejected": -0.0886012390255928, | |
| "sft_loss": 0.7357559204101562, | |
| "step": 1800 | |
| }, | |
| { | |
| "epoch": 2.92584360476864, | |
| "grad_norm": 2.13761043548584, | |
| "learning_rate": 6.94610413078306e-09, | |
| "logits/chosen": -3.028646945953369, | |
| "logits/rejected": -3.105076789855957, | |
| "logps/chosen": -0.7331860661506653, | |
| "logps/rejected": -1.0410078763961792, | |
| "loss": 0.7943, | |
| "odds_ratio_loss": 0.6107637882232666, | |
| "rewards/accuracies": 0.574999988079071, | |
| "rewards/chosen": -0.07331861555576324, | |
| "rewards/margins": 0.03078218176960945, | |
| "rewards/rejected": -0.104100801050663, | |
| "sft_loss": 0.7331860661506653, | |
| "step": 1810 | |
| }, | |
| { | |
| "epoch": 2.942008486562942, | |
| "grad_norm": 2.224161148071289, | |
| "learning_rate": 4.14834473758563e-09, | |
| "logits/chosen": -3.0697426795959473, | |
| "logits/rejected": -3.098311185836792, | |
| "logps/chosen": -0.6795364618301392, | |
| "logps/rejected": -0.9198546409606934, | |
| "loss": 0.7354, | |
| "odds_ratio_loss": 0.5586915016174316, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": -0.06795365363359451, | |
| "rewards/margins": 0.02403181977570057, | |
| "rewards/rejected": -0.09198546409606934, | |
| "sft_loss": 0.6795364618301392, | |
| "step": 1820 | |
| }, | |
| { | |
| "epoch": 2.9581733683572438, | |
| "grad_norm": 1.7413092851638794, | |
| "learning_rate": 2.067282222230349e-09, | |
| "logits/chosen": -3.101163864135742, | |
| "logits/rejected": -3.1490769386291504, | |
| "logps/chosen": -0.69708651304245, | |
| "logps/rejected": -0.9820737838745117, | |
| "loss": 0.753, | |
| "odds_ratio_loss": 0.5595835447311401, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": -0.06970865279436111, | |
| "rewards/margins": 0.028498733416199684, | |
| "rewards/rejected": -0.09820737689733505, | |
| "sft_loss": 0.69708651304245, | |
| "step": 1830 | |
| }, | |
| { | |
| "epoch": 2.9743382501515456, | |
| "grad_norm": 9.5276517868042, | |
| "learning_rate": 7.035141727212979e-10, | |
| "logits/chosen": -3.111255168914795, | |
| "logits/rejected": -3.1476521492004395, | |
| "logps/chosen": -0.6813724040985107, | |
| "logps/rejected": -0.9502272605895996, | |
| "loss": 0.7369, | |
| "odds_ratio_loss": 0.555463969707489, | |
| "rewards/accuracies": 0.637499988079071, | |
| "rewards/chosen": -0.06813724339008331, | |
| "rewards/margins": 0.026885494589805603, | |
| "rewards/rejected": -0.09502272307872772, | |
| "sft_loss": 0.6813724040985107, | |
| "step": 1840 | |
| }, | |
| { | |
| "epoch": 2.9905031319458475, | |
| "grad_norm": 10.846762657165527, | |
| "learning_rate": 5.743220219761592e-11, | |
| "logits/chosen": -3.105236530303955, | |
| "logits/rejected": -3.1389710903167725, | |
| "logps/chosen": -0.8284826278686523, | |
| "logps/rejected": -1.0016568899154663, | |
| "loss": 0.8952, | |
| "odds_ratio_loss": 0.6669169068336487, | |
| "rewards/accuracies": 0.612500011920929, | |
| "rewards/chosen": -0.08284827321767807, | |
| "rewards/margins": 0.017317429184913635, | |
| "rewards/rejected": -0.1001657024025917, | |
| "sft_loss": 0.8284826278686523, | |
| "step": 1850 | |
| }, | |
| { | |
| "epoch": 2.9969690846635686, | |
| "step": 1854, | |
| "total_flos": 2.0970902870084813e+18, | |
| "train_loss": 0.8330582246554065, | |
| "train_runtime": 34111.2463, | |
| "train_samples_per_second": 0.871, | |
| "train_steps_per_second": 0.054 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 1854, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 3, | |
| "save_steps": 500, | |
| "total_flos": 2.0970902870084813e+18, | |
| "train_batch_size": 2, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |