| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 10.0, |
| "eval_steps": 500, |
| "global_step": 620, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "epoch": 0.16227180527383367, |
| "grad_norm": 84.16447448730469, |
| "kl": 0.7359374761581421, |
| "learning_rate": 7.258064516129032e-08, |
| "logits/chosen": -36952473.6, |
| "logits/rejected": -292326604.8, |
| "logps/chosen": -899.2125, |
| "logps/rejected": -619.225, |
| "loss": 0.494, |
| "rewards/chosen": 0.0840850830078125, |
| "rewards/margins": 0.06186370849609375, |
| "rewards/rejected": 0.02222137451171875, |
| "step": 10 |
| }, |
| { |
| "epoch": 0.32454361054766734, |
| "grad_norm": 60.70732498168945, |
| "kl": 0.30781251192092896, |
| "learning_rate": 1.532258064516129e-07, |
| "logits/chosen": -46751744.0, |
| "logits/rejected": -247562649.6, |
| "logps/chosen": -1051.1125, |
| "logps/rejected": -544.9375, |
| "loss": 0.4834, |
| "rewards/chosen": 0.0416412353515625, |
| "rewards/margins": 0.09996337890625, |
| "rewards/rejected": -0.0583221435546875, |
| "step": 20 |
| }, |
| { |
| "epoch": 0.486815415821501, |
| "grad_norm": 43.18465042114258, |
| "kl": 0.4087890684604645, |
| "learning_rate": 2.3387096774193546e-07, |
| "logits/chosen": -55412326.4, |
| "logits/rejected": -207881830.4, |
| "logps/chosen": -937.7375, |
| "logps/rejected": -543.55, |
| "loss": 0.4162, |
| "rewards/chosen": 0.17140350341796876, |
| "rewards/margins": 1861222.9714035036, |
| "rewards/rejected": -1861222.8, |
| "step": 30 |
| }, |
| { |
| "epoch": 0.6490872210953347, |
| "grad_norm": 48.49949264526367, |
| "kl": 6.178906440734863, |
| "learning_rate": 3.1451612903225806e-07, |
| "logits/chosen": -53701836.8, |
| "logits/rejected": -276274380.8, |
| "logps/chosen": -980.5875, |
| "logps/rejected": -471.225, |
| "loss": 0.327, |
| "rewards/chosen": 1.36524658203125, |
| "rewards/margins": 2.0919921875, |
| "rewards/rejected": -0.72674560546875, |
| "step": 40 |
| }, |
| { |
| "epoch": 0.8113590263691683, |
| "grad_norm": 28.66044807434082, |
| "kl": 28.497461318969727, |
| "learning_rate": 3.951612903225806e-07, |
| "logits/chosen": -39727513.6, |
| "logits/rejected": -272444620.8, |
| "logps/chosen": -1023.975, |
| "logps/rejected": -532.5125, |
| "loss": 0.2801, |
| "rewards/chosen": 3.29287109375, |
| "rewards/margins": -253437.48212890624, |
| "rewards/rejected": 253440.775, |
| "step": 50 |
| }, |
| { |
| "epoch": 0.973630831643002, |
| "grad_norm": 17.18927764892578, |
| "kl": 73.3125, |
| "learning_rate": 4.758064516129032e-07, |
| "logits/chosen": -54368870.4, |
| "logits/rejected": -200237465.6, |
| "logps/chosen": -853.9375, |
| "logps/rejected": -491.55, |
| "loss": 0.2727, |
| "rewards/chosen": 8.0046875, |
| "rewards/margins": 2261238.6046875003, |
| "rewards/rejected": -2261230.6, |
| "step": 60 |
| }, |
| { |
| "epoch": 1.129817444219067, |
| "grad_norm": 9.472125053405762, |
| "kl": 98.43506622314453, |
| "learning_rate": 4.998058751077704e-07, |
| "logits/chosen": -20553196.051948052, |
| "logits/rejected": -251854422.44155845, |
| "logps/chosen": -842.7792207792207, |
| "logps/rejected": -573.1753246753246, |
| "loss": 0.2091, |
| "rewards/chosen": 9.786525974025974, |
| "rewards/margins": 5.313108766233766, |
| "rewards/rejected": 4.473417207792208, |
| "step": 70 |
| }, |
| { |
| "epoch": 1.2920892494929006, |
| "grad_norm": 8.546073913574219, |
| "kl": 122.63749694824219, |
| "learning_rate": 4.98855784989581e-07, |
| "logits/chosen": -29129113.6, |
| "logits/rejected": -209353113.6, |
| "logps/chosen": -882.325, |
| "logps/rejected": -479.55, |
| "loss": 0.1937, |
| "rewards/chosen": 11.08515625, |
| "rewards/margins": 4.6421875, |
| "rewards/rejected": 6.44296875, |
| "step": 80 |
| }, |
| { |
| "epoch": 1.4543610547667343, |
| "grad_norm": 8.500472068786621, |
| "kl": 121.0999984741211, |
| "learning_rate": 4.971170810820278e-07, |
| "logits/chosen": -40557363.2, |
| "logits/rejected": -195141632.0, |
| "logps/chosen": -795.875, |
| "logps/rejected": -480.8625, |
| "loss": 0.1752, |
| "rewards/chosen": 11.70703125, |
| "rewards/margins": 1535225.60703125, |
| "rewards/rejected": -1535213.9, |
| "step": 90 |
| }, |
| { |
| "epoch": 1.616632860040568, |
| "grad_norm": 12.312355995178223, |
| "kl": 109.1312484741211, |
| "learning_rate": 4.945952732665755e-07, |
| "logits/chosen": -8305868.8, |
| "logits/rejected": -211968409.6, |
| "logps/chosen": -808.3, |
| "logps/rejected": -387.16875, |
| "loss": 0.1271, |
| "rewards/chosen": 11.8703125, |
| "rewards/margins": 8.137158203125, |
| "rewards/rejected": 3.733154296875, |
| "step": 100 |
| }, |
| { |
| "epoch": 1.7789046653144016, |
| "grad_norm": 7.883743762969971, |
| "kl": 109.5562515258789, |
| "learning_rate": 4.912983530488966e-07, |
| "logits/chosen": -37431603.2, |
| "logits/rejected": -249792102.4, |
| "logps/chosen": -916.45625, |
| "logps/rejected": -489.2125, |
| "loss": 0.147, |
| "rewards/chosen": 11.49296875, |
| "rewards/margins": -253333.48203125, |
| "rewards/rejected": 253344.975, |
| "step": 110 |
| }, |
| { |
| "epoch": 1.9411764705882353, |
| "grad_norm": 3.3018791675567627, |
| "kl": 114.1500015258789, |
| "learning_rate": 4.872367682341172e-07, |
| "logits/chosen": -64476160.0, |
| "logits/rejected": -196329881.6, |
| "logps/chosen": -797.3875, |
| "logps/rejected": -489.9875, |
| "loss": 0.1421, |
| "rewards/chosen": 12.90703125, |
| "rewards/margins": 9.448437499999999, |
| "rewards/rejected": 3.45859375, |
| "step": 120 |
| }, |
| { |
| "epoch": 2.0973630831643, |
| "grad_norm": 13.774515151977539, |
| "kl": 92.26298522949219, |
| "learning_rate": 4.824233898181509e-07, |
| "logits/chosen": -42439241.14285714, |
| "logits/rejected": -251607172.98701298, |
| "logps/chosen": -861.1233766233767, |
| "logps/rejected": -612.8311688311688, |
| "loss": 0.1248, |
| "rewards/chosen": 11.994318181818182, |
| "rewards/margins": 2648996.7215909087, |
| "rewards/rejected": -2648984.727272727, |
| "step": 130 |
| }, |
| { |
| "epoch": 2.259634888438134, |
| "grad_norm": 11.28150463104248, |
| "kl": 78.5999984741211, |
| "learning_rate": 4.76873471200039e-07, |
| "logits/chosen": -43557273.6, |
| "logits/rejected": -245417984.0, |
| "logps/chosen": -808.3875, |
| "logps/rejected": -577.90625, |
| "loss": 0.1185, |
| "rewards/chosen": 9.87734375, |
| "rewards/margins": 9.4081298828125, |
| "rewards/rejected": 0.4692138671875, |
| "step": 140 |
| }, |
| { |
| "epoch": 2.4219066937119678, |
| "grad_norm": 6.963021755218506, |
| "kl": 72.09375, |
| "learning_rate": 4.7060459984455474e-07, |
| "logits/chosen": -70427648.0, |
| "logits/rejected": -230709657.6, |
| "logps/chosen": -924.20625, |
| "logps/rejected": -505.775, |
| "loss": 0.1058, |
| "rewards/chosen": 10.9953125, |
| "rewards/margins": 1436882.7953125, |
| "rewards/rejected": -1436871.8, |
| "step": 150 |
| }, |
| { |
| "epoch": 2.584178498985801, |
| "grad_norm": 11.5512056350708, |
| "kl": 59.21875, |
| "learning_rate": 4.6363664154824735e-07, |
| "logits/chosen": -75585536.0, |
| "logits/rejected": -257813708.8, |
| "logps/chosen": -759.025, |
| "logps/rejected": -496.0625, |
| "loss": 0.095, |
| "rewards/chosen": 10.133203125, |
| "rewards/margins": 11.517364501953125, |
| "rewards/rejected": -1.384161376953125, |
| "step": 160 |
| }, |
| { |
| "epoch": 2.7464503042596347, |
| "grad_norm": 16.846378326416016, |
| "kl": 42.296875, |
| "learning_rate": 4.5599167748554633e-07, |
| "logits/chosen": -99328000.0, |
| "logits/rejected": -318544281.6, |
| "logps/chosen": -964.01875, |
| "logps/rejected": -570.0125, |
| "loss": 0.1034, |
| "rewards/chosen": 8.240234375, |
| "rewards/margins": 304753.265234375, |
| "rewards/rejected": -304745.025, |
| "step": 170 |
| }, |
| { |
| "epoch": 2.9087221095334685, |
| "grad_norm": 7.493341445922852, |
| "kl": 31.861719131469727, |
| "learning_rate": 4.476939342344246e-07, |
| "logits/chosen": -141419724.8, |
| "logits/rejected": -281131417.6, |
| "logps/chosen": -841.39375, |
| "logps/rejected": -558.1875, |
| "loss": 0.0987, |
| "rewards/chosen": 7.83671875, |
| "rewards/margins": -186562.66328125, |
| "rewards/rejected": 186570.5, |
| "step": 180 |
| }, |
| { |
| "epoch": 3.0649087221095335, |
| "grad_norm": 7.3059892654418945, |
| "kl": 38.8863639831543, |
| "learning_rate": 4.387697070033649e-07, |
| "logits/chosen": -149665499.42857143, |
| "logits/rejected": -332706696.3116883, |
| "logps/chosen": -899.012987012987, |
| "logps/rejected": -628.8181818181819, |
| "loss": 0.0836, |
| "rewards/chosen": 10.13271103896104, |
| "rewards/margins": 2618391.2236201297, |
| "rewards/rejected": -2618381.090909091, |
| "step": 190 |
| }, |
| { |
| "epoch": 3.227180527383367, |
| "grad_norm": 3.7938599586486816, |
| "kl": 54.193748474121094, |
| "learning_rate": 4.2924727630292124e-07, |
| "logits/chosen": -99653632.0, |
| "logits/rejected": -326130892.8, |
| "logps/chosen": -845.28125, |
| "logps/rejected": -579.2875, |
| "loss": 0.0948, |
| "rewards/chosen": 9.30859375, |
| "rewards/margins": 11.3248046875, |
| "rewards/rejected": -2.0162109375, |
| "step": 200 |
| }, |
| { |
| "epoch": 3.389452332657201, |
| "grad_norm": 6.6560492515563965, |
| "kl": 59.662498474121094, |
| "learning_rate": 4.1915681832593934e-07, |
| "logits/chosen": -138940006.4, |
| "logits/rejected": -283273625.6, |
| "logps/chosen": -865.6625, |
| "logps/rejected": -578.7625, |
| "loss": 0.0752, |
| "rewards/chosen": 10.1765625, |
| "rewards/margins": 1564678.5765624999, |
| "rewards/rejected": -1564668.4, |
| "step": 210 |
| }, |
| { |
| "epoch": 3.5517241379310347, |
| "grad_norm": 7.188765525817871, |
| "kl": 39.96953201293945, |
| "learning_rate": 4.085303093204377e-07, |
| "logits/chosen": -145563238.4, |
| "logits/rejected": -305337139.2, |
| "logps/chosen": -854.525, |
| "logps/rejected": -579.0125, |
| "loss": 0.0652, |
| "rewards/chosen": 9.2734375, |
| "rewards/margins": 12.3808837890625, |
| "rewards/rejected": -3.1074462890625, |
| "step": 220 |
| }, |
| { |
| "epoch": 3.713995943204868, |
| "grad_norm": 9.175395011901855, |
| "kl": 26.307811737060547, |
| "learning_rate": 3.974014242581871e-07, |
| "logits/chosen": -173807206.4, |
| "logits/rejected": -361424486.4, |
| "logps/chosen": -956.9375, |
| "logps/rejected": -474.825, |
| "loss": 0.0615, |
| "rewards/chosen": 7.969140625, |
| "rewards/margins": 536591.069140625, |
| "rewards/rejected": -536583.1, |
| "step": 230 |
| }, |
| { |
| "epoch": 3.8762677484787016, |
| "grad_norm": 7.2789306640625, |
| "kl": 35.146873474121094, |
| "learning_rate": 3.858054301201047e-07, |
| "logits/chosen": -169469542.4, |
| "logits/rejected": -343206297.6, |
| "logps/chosen": -873.7625, |
| "logps/rejected": -612.525, |
| "loss": 0.0585, |
| "rewards/chosen": 8.37109375, |
| "rewards/margins": -114471.84140625, |
| "rewards/rejected": 114480.2125, |
| "step": 240 |
| }, |
| { |
| "epoch": 4.032454361054767, |
| "grad_norm": 7.370634078979492, |
| "kl": 37.38230514526367, |
| "learning_rate": 3.7377907413663576e-07, |
| "logits/chosen": -175320077.2987013, |
| "logits/rejected": -325758178.0779221, |
| "logps/chosen": -908.3181818181819, |
| "logps/rejected": -593.3116883116883, |
| "loss": 0.0588, |
| "rewards/chosen": 9.853896103896103, |
| "rewards/margins": 2647334.529220779, |
| "rewards/rejected": -2647324.675324675, |
| "step": 250 |
| }, |
| { |
| "epoch": 4.1947261663286, |
| "grad_norm": 20.8977108001709, |
| "kl": 22.143749237060547, |
| "learning_rate": 3.613604673372861e-07, |
| "logits/chosen": -160774553.6, |
| "logits/rejected": -361404825.6, |
| "logps/chosen": -796.53125, |
| "logps/rejected": -669.5625, |
| "loss": 0.0639, |
| "rewards/chosen": 8.620703125, |
| "rewards/margins": 12.99384765625, |
| "rewards/rejected": -4.37314453125, |
| "step": 260 |
| }, |
| { |
| "epoch": 4.356997971602434, |
| "grad_norm": 8.267151832580566, |
| "kl": 14.619531631469727, |
| "learning_rate": 3.4858896377832965e-07, |
| "logits/chosen": -194864332.8, |
| "logits/rejected": -343074406.4, |
| "logps/chosen": -909.8375, |
| "logps/rejected": -613.9625, |
| "loss": 0.0528, |
| "rewards/chosen": 8.049609375, |
| "rewards/margins": 1472961.049609375, |
| "rewards/rejected": -1472953.0, |
| "step": 270 |
| }, |
| { |
| "epoch": 4.519269776876268, |
| "grad_norm": 6.327075004577637, |
| "kl": 20.830469131469727, |
| "learning_rate": 3.355050358314172e-07, |
| "logits/chosen": -216122982.4, |
| "logits/rejected": -317764403.2, |
| "logps/chosen": -820.175, |
| "logps/rejected": -619.325, |
| "loss": 0.0507, |
| "rewards/chosen": 8.112890625, |
| "rewards/margins": 13.183007812500001, |
| "rewards/rejected": -5.0701171875, |
| "step": 280 |
| }, |
| { |
| "epoch": 4.681541582150102, |
| "grad_norm": 3.9251317977905273, |
| "kl": 18.582813262939453, |
| "learning_rate": 3.2215014592828766e-07, |
| "logits/chosen": -225445478.4, |
| "logits/rejected": -377572556.8, |
| "logps/chosen": -922.80625, |
| "logps/rejected": -511.4125, |
| "loss": 0.0355, |
| "rewards/chosen": 8.7359375, |
| "rewards/margins": 684053.0359375001, |
| "rewards/rejected": -684044.3, |
| "step": 290 |
| }, |
| { |
| "epoch": 4.8438133874239355, |
| "grad_norm": 2.7528088092803955, |
| "kl": 32.806251525878906, |
| "learning_rate": 3.085666151680205e-07, |
| "logits/chosen": -194420736.0, |
| "logits/rejected": -402663014.4, |
| "logps/chosen": -906.09375, |
| "logps/rejected": -579.35, |
| "loss": 0.0304, |
| "rewards/chosen": 9.00625, |
| "rewards/margins": 22823.6671875, |
| "rewards/rejected": -22814.6609375, |
| "step": 300 |
| }, |
| { |
| "epoch": 5.0, |
| "grad_norm": 4.467460632324219, |
| "kl": 34.149349212646484, |
| "learning_rate": 2.947974892032063e-07, |
| "logits/chosen": -231085042.7012987, |
| "logits/rejected": -335724756.77922076, |
| "logps/chosen": -854.9805194805194, |
| "logps/rejected": -645.4155844155844, |
| "loss": 0.0361, |
| "rewards/chosen": 10.591720779220779, |
| "rewards/margins": 2898001.8644480524, |
| "rewards/rejected": -2897991.272727273, |
| "step": 310 |
| }, |
| { |
| "epoch": 5.162271805273834, |
| "grad_norm": 15.31023120880127, |
| "kl": 17.678905487060547, |
| "learning_rate": 2.8088640183003664e-07, |
| "logits/chosen": -220064153.6, |
| "logits/rejected": -414279270.4, |
| "logps/chosen": -786.05, |
| "logps/rejected": -687.275, |
| "loss": 0.0426, |
| "rewards/chosen": 8.684375, |
| "rewards/margins": 13.99609375, |
| "rewards/rejected": -5.31171875, |
| "step": 320 |
| }, |
| { |
| "epoch": 5.324543610547668, |
| "grad_norm": 12.307633399963379, |
| "kl": 4.917187690734863, |
| "learning_rate": 2.668774367145913e-07, |
| "logits/chosen": -235716608.0, |
| "logits/rejected": -369033216.0, |
| "logps/chosen": -952.45, |
| "logps/rejected": -636.8875, |
| "loss": 0.0409, |
| "rewards/chosen": 7.369140625, |
| "rewards/margins": 14.8421875, |
| "rewards/rejected": -7.473046875, |
| "step": 330 |
| }, |
| { |
| "epoch": 5.486815415821501, |
| "grad_norm": 5.51188850402832, |
| "kl": 5.789843559265137, |
| "learning_rate": 2.5281498769350646e-07, |
| "logits/chosen": -236973260.8, |
| "logits/rejected": -331428659.2, |
| "logps/chosen": -838.6375, |
| "logps/rejected": -630.125, |
| "loss": 0.0264, |
| "rewards/chosen": 7.7671875, |
| "rewards/margins": 1459860.0671875, |
| "rewards/rejected": -1459852.3, |
| "step": 340 |
| }, |
| { |
| "epoch": 5.649087221095335, |
| "grad_norm": 14.79988956451416, |
| "kl": 5.79296875, |
| "learning_rate": 2.3874361809172427e-07, |
| "logits/chosen": -255219302.4, |
| "logits/rejected": -402902220.8, |
| "logps/chosen": -896.85, |
| "logps/rejected": -556.4, |
| "loss": 0.0206, |
| "rewards/chosen": 8.0015625, |
| "rewards/margins": 15.924218750000001, |
| "rewards/rejected": -7.92265625, |
| "step": 350 |
| }, |
| { |
| "epoch": 5.811359026369169, |
| "grad_norm": 1.1963369846343994, |
| "kl": 5.693749904632568, |
| "learning_rate": 2.2470791950314196e-07, |
| "logits/chosen": -237377945.6, |
| "logits/rejected": -411195801.6, |
| "logps/chosen": -967.6, |
| "logps/rejected": -634.9125, |
| "loss": 0.0207, |
| "rewards/chosen": 7.40546875, |
| "rewards/margins": 891114.00546875, |
| "rewards/rejected": -891106.6, |
| "step": 360 |
| }, |
| { |
| "epoch": 5.973630831643002, |
| "grad_norm": 0.8492076396942139, |
| "kl": 14.582812309265137, |
| "learning_rate": 2.1075237048167677e-07, |
| "logits/chosen": -262532300.8, |
| "logits/rejected": -366804992.0, |
| "logps/chosen": -832.14375, |
| "logps/rejected": -621.4625, |
| "loss": 0.0228, |
| "rewards/chosen": 9.843359375, |
| "rewards/margins": 3079304.6433593747, |
| "rewards/rejected": -3079294.8, |
| "step": 370 |
| }, |
| { |
| "epoch": 6.129817444219067, |
| "grad_norm": 14.829185485839844, |
| "kl": 13.705357551574707, |
| "learning_rate": 1.96921195590551e-07, |
| "logits/chosen": -247062208.83116883, |
| "logits/rejected": -430358740.77922076, |
| "logps/chosen": -851.1038961038961, |
| "logps/rejected": -713.3376623376623, |
| "loss": 0.0316, |
| "rewards/chosen": 9.212256493506494, |
| "rewards/margins": 15.109577922077923, |
| "rewards/rejected": -5.897321428571429, |
| "step": 380 |
| }, |
| { |
| "epoch": 6.292089249492901, |
| "grad_norm": 13.538140296936035, |
| "kl": 4.005468845367432, |
| "learning_rate": 1.8325822525646205e-07, |
| "logits/chosen": -268140544.0, |
| "logits/rejected": -391171276.8, |
| "logps/chosen": -932.825, |
| "logps/rejected": -651.425, |
| "loss": 0.0245, |
| "rewards/chosen": 7.41875, |
| "rewards/margins": 14.955078125, |
| "rewards/rejected": -7.536328125, |
| "step": 390 |
| }, |
| { |
| "epoch": 6.454361054766734, |
| "grad_norm": 3.61026930809021, |
| "kl": 2.526562452316284, |
| "learning_rate": 1.6980675687275614e-07, |
| "logits/chosen": -282066944.0, |
| "logits/rejected": -389788467.2, |
| "logps/chosen": -851.1, |
| "logps/rejected": -648.5375, |
| "loss": 0.02, |
| "rewards/chosen": 7.39765625, |
| "rewards/margins": 1656471.09765625, |
| "rewards/rejected": -1656463.7, |
| "step": 400 |
| }, |
| { |
| "epoch": 6.616632860040568, |
| "grad_norm": 0.35545381903648376, |
| "kl": 6.8046875, |
| "learning_rate": 1.5660941759176157e-07, |
| "logits/chosen": -249454592.0, |
| "logits/rejected": -395798118.4, |
| "logps/chosen": -850.96875, |
| "logps/rejected": -529.025, |
| "loss": 0.0107, |
| "rewards/chosen": 8.4625, |
| "rewards/margins": 16.0609375, |
| "rewards/rejected": -7.5984375, |
| "step": 410 |
| }, |
| { |
| "epoch": 6.778904665314402, |
| "grad_norm": 0.6669896245002747, |
| "kl": 8.045312881469727, |
| "learning_rate": 1.4370802924108988e-07, |
| "logits/chosen": -258266726.4, |
| "logits/rejected": -430876262.4, |
| "logps/chosen": -961.0375, |
| "logps/rejected": -622.3625, |
| "loss": 0.0184, |
| "rewards/chosen": 8.0828125, |
| "rewards/margins": 1007446.3828125, |
| "rewards/rejected": -1007438.3, |
| "step": 420 |
| }, |
| { |
| "epoch": 6.9411764705882355, |
| "grad_norm": 0.8302009701728821, |
| "kl": 6.962500095367432, |
| "learning_rate": 1.3114347579197618e-07, |
| "logits/chosen": -283987148.8, |
| "logits/rejected": -371962675.2, |
| "logps/chosen": -852.0, |
| "logps/rejected": -631.225, |
| "loss": 0.0145, |
| "rewards/chosen": 8.53125, |
| "rewards/margins": 16.691015625, |
| "rewards/rejected": -8.159765625, |
| "step": 430 |
| }, |
| { |
| "epoch": 7.0973630831643, |
| "grad_norm": 0.6516138911247253, |
| "kl": 10.787337303161621, |
| "learning_rate": 1.1895557379965004e-07, |
| "logits/chosen": -267412413.5064935, |
| "logits/rejected": -424823076.5714286, |
| "logps/chosen": -890.7727272727273, |
| "logps/rejected": -724.6883116883117, |
| "loss": 0.0223, |
| "rewards/chosen": 9.612012987012987, |
| "rewards/margins": 3236774.702922078, |
| "rewards/rejected": -3236765.090909091, |
| "step": 440 |
| }, |
| { |
| "epoch": 7.259634888438134, |
| "grad_norm": 21.117584228515625, |
| "kl": 12.122655868530273, |
| "learning_rate": 1.0718294622630186e-07, |
| "logits/chosen": -270696448.0, |
| "logits/rejected": -412673638.4, |
| "logps/chosen": -829.05, |
| "logps/rejected": -671.0625, |
| "loss": 0.0188, |
| "rewards/chosen": 8.44765625, |
| "rewards/margins": 14.951171875, |
| "rewards/rejected": -6.503515625, |
| "step": 450 |
| }, |
| { |
| "epoch": 7.421906693711968, |
| "grad_norm": 3.9178707599639893, |
| "kl": 4.1484375, |
| "learning_rate": 9.586290004649867e-08, |
| "logits/chosen": -286366105.6, |
| "logits/rejected": -389460787.2, |
| "logps/chosen": -958.95625, |
| "logps/rejected": -601.0625, |
| "loss": 0.0151, |
| "rewards/chosen": 8.26640625, |
| "rewards/margins": 1740058.6664062499, |
| "rewards/rejected": -1740050.4, |
| "step": 460 |
| }, |
| { |
| "epoch": 7.584178498985802, |
| "grad_norm": 2.008082151412964, |
| "kl": 2.745312452316284, |
| "learning_rate": 8.503130802290861e-08, |
| "logits/chosen": -276156416.0, |
| "logits/rejected": -394657792.0, |
| "logps/chosen": -786.85, |
| "logps/rejected": -588.85, |
| "loss": 0.0099, |
| "rewards/chosen": 7.89765625, |
| "rewards/margins": 16.633984375, |
| "rewards/rejected": -8.736328125, |
| "step": 470 |
| }, |
| { |
| "epoch": 7.746450304259635, |
| "grad_norm": 0.24131633341312408, |
| "kl": 5.1953125, |
| "learning_rate": 7.472249502698685e-08, |
| "logits/chosen": -269829734.4, |
| "logits/rejected": -436561510.4, |
| "logps/chosen": -967.49375, |
| "logps/rejected": -638.0625, |
| "loss": 0.0157, |
| "rewards/chosen": 7.919921875, |
| "rewards/margins": 883133.219921875, |
| "rewards/rejected": -883125.3, |
| "step": 480 |
| }, |
| { |
| "epoch": 7.9087221095334685, |
| "grad_norm": 2.947861909866333, |
| "kl": 6.618750095367432, |
| "learning_rate": 6.496912926486278e-08, |
| "logits/chosen": -284753920.0, |
| "logits/rejected": -377408716.8, |
| "logps/chosen": -833.10625, |
| "logps/rejected": -611.9375, |
| "loss": 0.0083, |
| "rewards/chosen": 8.5046875, |
| "rewards/margins": 183315.91718750002, |
| "rewards/rejected": -183307.4125, |
| "step": 490 |
| }, |
| { |
| "epoch": 8.064908722109534, |
| "grad_norm": 5.739405155181885, |
| "kl": 9.308441162109375, |
| "learning_rate": 5.580211875313345e-08, |
| "logits/chosen": -287616226.0779221, |
| "logits/rejected": -422971046.23376626, |
| "logps/chosen": -899.9415584415584, |
| "logps/rejected": -680.6233766233767, |
| "loss": 0.0178, |
| "rewards/chosen": 10.097402597402597, |
| "rewards/margins": 3274245.9415584416, |
| "rewards/rejected": -3274235.8441558443, |
| "step": 500 |
| }, |
| { |
| "epoch": 8.227180527383368, |
| "grad_norm": 6.1757097244262695, |
| "kl": 10.469531059265137, |
| "learning_rate": 4.725051337262475e-08, |
| "logits/chosen": -251002880.0, |
| "logits/rejected": -420275814.4, |
| "logps/chosen": -854.85625, |
| "logps/rejected": -645.3125, |
| "loss": 0.0152, |
| "rewards/chosen": 8.628125, |
| "rewards/margins": 15.3453125, |
| "rewards/rejected": -6.7171875, |
| "step": 510 |
| }, |
| { |
| "epoch": 8.3894523326572, |
| "grad_norm": 5.071248531341553, |
| "kl": 7.877343654632568, |
| "learning_rate": 3.934141281050909e-08, |
| "logits/chosen": -291821977.6, |
| "logits/rejected": -385135411.2, |
| "logps/chosen": -886.45, |
| "logps/rejected": -652.85, |
| "loss": 0.0155, |
| "rewards/chosen": 8.568359375, |
| "rewards/margins": 1805589.568359375, |
| "rewards/rejected": -1805581.0, |
| "step": 520 |
| }, |
| { |
| "epoch": 8.551724137931034, |
| "grad_norm": 0.6050207614898682, |
| "kl": 5.170312404632568, |
| "learning_rate": 3.209988068250688e-08, |
| "logits/chosen": -285185638.4, |
| "logits/rejected": -395824332.8, |
| "logps/chosen": -863.9625, |
| "logps/rejected": -640.1625, |
| "loss": 0.0084, |
| "rewards/chosen": 8.50703125, |
| "rewards/margins": 16.423046875, |
| "rewards/rejected": -7.916015625, |
| "step": 530 |
| }, |
| { |
| "epoch": 8.713995943204868, |
| "grad_norm": 2.260059118270874, |
| "kl": 3.5218749046325684, |
| "learning_rate": 2.5548865107314604e-08, |
| "logits/chosen": -286359552.0, |
| "logits/rejected": -428421939.2, |
| "logps/chosen": -954.11875, |
| "logps/rejected": -522.375, |
| "loss": 0.0129, |
| "rewards/chosen": 8.103125, |
| "rewards/margins": 902798.203125, |
| "rewards/rejected": -902790.1, |
| "step": 540 |
| }, |
| { |
| "epoch": 8.876267748478702, |
| "grad_norm": 1.0818209648132324, |
| "kl": 4.739062309265137, |
| "learning_rate": 1.970912598495689e-08, |
| "logits/chosen": -280064819.2, |
| "logits/rejected": -416330547.2, |
| "logps/chosen": -878.925, |
| "logps/rejected": -671.425, |
| "loss": 0.008, |
| "rewards/chosen": 7.967578125, |
| "rewards/margins": 222435.842578125, |
| "rewards/rejected": -222427.875, |
| "step": 550 |
| }, |
| { |
| "epoch": 9.032454361054766, |
| "grad_norm": 2.1863319873809814, |
| "kl": 5.1055192947387695, |
| "learning_rate": 1.4599169209513568e-08, |
| "logits/chosen": -289883601.45454544, |
| "logits/rejected": -403381740.0519481, |
| "logps/chosen": -919.012987012987, |
| "logps/rejected": -651.961038961039, |
| "loss": 0.0142, |
| "rewards/chosen": 8.921266233766234, |
| "rewards/margins": 3287868.6615259736, |
| "rewards/rejected": -3287859.74025974, |
| "step": 560 |
| }, |
| { |
| "epoch": 9.1947261663286, |
| "grad_norm": 2.410034656524658, |
| "kl": 4.432812690734863, |
| "learning_rate": 1.023518802469947e-08, |
| "logits/chosen": -269074432.0, |
| "logits/rejected": -430492876.8, |
| "logps/chosen": -797.04375, |
| "logps/rejected": -712.25, |
| "loss": 0.0147, |
| "rewards/chosen": 8.553125, |
| "rewards/margins": 15.996875, |
| "rewards/rejected": -7.44375, |
| "step": 570 |
| }, |
| { |
| "epoch": 9.356997971602434, |
| "grad_norm": 1.298397421836853, |
| "kl": 5.707812309265137, |
| "learning_rate": 6.631011708138207e-09, |
| "logits/chosen": -292247961.6, |
| "logits/rejected": -404599603.2, |
| "logps/chosen": -903.8875, |
| "logps/rejected": -643.45, |
| "loss": 0.0152, |
| "rewards/chosen": 8.49375, |
| "rewards/margins": 1818702.09375, |
| "rewards/rejected": -1818693.6, |
| "step": 580 |
| }, |
| { |
| "epoch": 9.519269776876268, |
| "grad_norm": 0.7316223382949829, |
| "kl": 5.643750190734863, |
| "learning_rate": 3.798061746947995e-09, |
| "logits/chosen": -298290380.8, |
| "logits/rejected": -368705536.0, |
| "logps/chosen": -820.0125, |
| "logps/rejected": -653.4375, |
| "loss": 0.009, |
| "rewards/chosen": 8.12421875, |
| "rewards/margins": 15.853515625, |
| "rewards/rejected": -7.729296875, |
| "step": 590 |
| }, |
| { |
| "epoch": 9.681541582150102, |
| "grad_norm": 0.2766137421131134, |
| "kl": 4.55859375, |
| "learning_rate": 1.7453156435165983e-09, |
| "logits/chosen": -298279731.2, |
| "logits/rejected": -418748825.6, |
| "logps/chosen": -924.475, |
| "logps/rejected": -543.5875, |
| "loss": 0.0058, |
| "rewards/chosen": 8.64140625, |
| "rewards/margins": 902797.3414062499, |
| "rewards/rejected": -902788.7, |
| "step": 600 |
| }, |
| { |
| "epoch": 9.843813387423936, |
| "grad_norm": 0.1658095270395279, |
| "kl": 5.104687690734863, |
| "learning_rate": 4.792784661642458e-10, |
| "logits/chosen": -260448256.0, |
| "logits/rejected": -439471308.8, |
| "logps/chosen": -921.39375, |
| "logps/rejected": -623.55, |
| "loss": 0.0136, |
| "rewards/chosen": 7.892578125, |
| "rewards/margins": 219158.467578125, |
| "rewards/rejected": -219150.575, |
| "step": 610 |
| }, |
| { |
| "epoch": 10.0, |
| "grad_norm": 0.4314670264720917, |
| "kl": 6.086039066314697, |
| "learning_rate": 3.962234848359225e-12, |
| "logits/chosen": -294411543.27272725, |
| "logits/rejected": -375213175.6883117, |
| "logps/chosen": -868.5064935064935, |
| "logps/rejected": -691.6623376623377, |
| "loss": 0.0102, |
| "rewards/chosen": 9.39163961038961, |
| "rewards/margins": 3321915.885146104, |
| "rewards/rejected": -3321906.4935064935, |
| "step": 620 |
| } |
| ], |
| "logging_steps": 10, |
| "max_steps": 620, |
| "num_input_tokens_seen": 0, |
| "num_train_epochs": 10, |
| "save_steps": 200, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
