Training in progress, step 100, checkpoint

checkpoint-100/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6b4d33389f1cc4a587fb1cc0e286e0fb0f48ea297e318d76ce7fc7dc4e15d4ac
+oid sha256:d46bbda891252d1bc7d7b207072338c59dd511f23a0bb1a77a233dd55bf64bc3
 size 1976163472

checkpoint-100/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:96d319c90d917e3d81f0107f270f4c2887636b656c752ce11241c1c9632e8d8b
+oid sha256:e61137fa392006841b3c9d3b0075e477b2bfd4da9800e8738d48e950a612ad64
 size 3952505274

checkpoint-100/rng_state_0.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d952ecf17d9b51ed5e329f95c84c1249d0107fc4d0109eb87fb465326e8bff24
+oid sha256:e8016113faecd368858eb6ebc4fcb61a9f1956107a452dada8e98bfda76288f6
 size 15024

checkpoint-100/rng_state_1.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3dcf2b47c9a735f13280e89c61a0263a149b2cdaf95d263753b5201739a39512
+oid sha256:974125c47b4b1edfbbcd3caeae8511abb215ef1494e888b05e7fde18c3ed08ed
 size 15024

checkpoint-100/rng_state_2.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:006dea8e4200032c17f87db4801d9905390e53a088c7e9a09835fe1319c7bc83
+oid sha256:9ac54bed7d9d2e9cb07f31f09406c3bf48fced29450844d31d747045d9f2f6ea
 size 15024

checkpoint-100/rng_state_3.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cc83440fa410c77e3d675e7539df9956319717fb0fd9b5adaf4caf1f6ef19a1c
+oid sha256:817aecd630d55ef1ec1da71dbd2daae7b12d4db119f38d05d48fdc7f0cb7134f
 size 15024

checkpoint-100/trainer_state.json

@@ -9,7 +9,7 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 134.4375,
+      "completion_length": 25.8125,
       "epoch": 0.0013333333333333333,
       "grad_norm": 0.0,
       "kl": 0.0,
@@ -22,23 +22,23 @@
       "step": 1
     },
     {
-      "completion_length": 157.375,
+      "completion_length": 45.0,
       "epoch": 0.0026666666666666666,
-      "grad_norm": 0.0,
+      "grad_norm": 9.964546203613281,
       "kl": 0.0,
       "learning_rate": 4.995066821070679e-07,
-      "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "loss": -0.0,
+      "reward": 0.0625,
+      "reward_std": 0.125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.0625,
       "step": 2
     },
     {
-      "completion_length": 161.75,
+      "completion_length": 38.9375,
       "epoch": 0.004,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "grad_norm": 0.002867324510589242,
+      "kl": 0.0005742026260122657,
       "learning_rate": 4.9889049115077e-07,
       "loss": 0.0,
       "reward": 0.0,
@@ -48,10 +48,10 @@
       "step": 3
     },
     {
-      "completion_length": 147.8125,
+      "completion_length": 27.4375,
       "epoch": 0.005333333333333333,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "grad_norm": 0.006247804034501314,
+      "kl": 0.0010731846559792757,
       "learning_rate": 4.980286753286194e-07,
       "loss": 0.0,
       "reward": 0.0,
@@ -61,23 +61,23 @@
       "step": 4
     },
     {
-      "completion_length": 207.5625,
+      "completion_length": 47.125,
       "epoch": 0.006666666666666667,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "grad_norm": 4.878363132476807,
+      "kl": 0.0029146450106054544,
       "learning_rate": 4.969220851487844e-07,
       "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "reward": 0.0625,
+      "reward_std": 0.125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.0625,
       "step": 5
     },
     {
-      "completion_length": 188.875,
+      "completion_length": 48.3125,
       "epoch": 0.008,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "grad_norm": 0.011296601966023445,
+      "kl": 0.0013835413847118616,
       "learning_rate": 4.955718126821722e-07,
       "loss": 0.0,
       "reward": 0.0,
@@ -87,23 +87,23 @@
       "step": 6
     },
     {
-      "completion_length": 197.5,
+      "completion_length": 64.5625,
       "epoch": 0.009333333333333334,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "grad_norm": 9.235820770263672,
+      "kl": 0.0008675489807501435,
       "learning_rate": 4.939791904846868e-07,
       "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "reward": 0.0625,
+      "reward_std": 0.125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.0625,
       "step": 7
     },
     {
-      "completion_length": 158.5625,
+      "completion_length": 40.6875,
       "epoch": 0.010666666666666666,
-      "grad_norm": 0.0,
-      "kl": 0.0,
+      "grad_norm": 0.01409083604812622,
+      "kl": 0.002122239675372839,
       "learning_rate": 4.921457902821578e-07,
       "loss": 0.0,
       "reward": 0.0,
@@ -113,10 +113,10 @@
       "step": 8
     },
     {
-      "completion_length": 149.5,
+      "completion_length": 55.0,
       "epoch": 0.012,
-      "grad_norm": 5.7435736656188965,
-      "kl": 0.0,
+      "grad_norm": 12.573073387145996,
+      "kl": 0.006520974449813366,
       "learning_rate": 4.900734214192358e-07,
       "loss": 0.0,
       "reward": 0.0625,
@@ -126,10 +126,10 @@
       "step": 9
     },
     {
-      "completion_length": 184.0625,
+      "completion_length": 83.875,
       "epoch": 0.013333333333333334,
-      "grad_norm": 2.868870973587036,
-      "kl": 0.0008592925732955337,
+      "grad_norm": 6.103325843811035,
+      "kl": 0.003683926770463586,
       "learning_rate": 4.877641290737883e-07,
       "loss": 0.0,
       "reward": 0.0625,
@@ -139,725 +139,725 @@
       "step": 10
     },
     {
-      "completion_length": 173.5,
+      "completion_length": 47.125,
       "epoch": 0.014666666666666666,
-      "grad_norm": 0.001981294248253107,
-      "kl": 0.000871832889970392,
+      "grad_norm": 6.05898380279541,
+      "kl": 0.010808728635311127,
       "learning_rate": 4.852201922385564e-07,
       "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "reward": 0.0625,
+      "reward_std": 0.125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.0625,
       "step": 11
     },
     {
-      "completion_length": 181.0625,
+      "completion_length": 52.75,
       "epoch": 0.016,
-      "grad_norm": 0.004562250338494778,
-      "kl": 0.001329638296738267,
+      "grad_norm": 10.170279502868652,
+      "kl": 0.017805274575948715,
       "learning_rate": 4.824441214720628e-07,
       "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "reward": 0.0625,
+      "reward_std": 0.125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.0625,
       "step": 12
     },
     {
-      "completion_length": 183.375,
+      "completion_length": 86.5,
       "epoch": 0.017333333333333333,
-      "grad_norm": 0.0010324150789529085,
-      "kl": 0.0008550028433091938,
+      "grad_norm": 3.2019193172454834,
+      "kl": 0.00332952244207263,
       "learning_rate": 4.794386564209952e-07,
       "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "reward": 0.0625,
+      "reward_std": 0.125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.0625,
       "step": 13
     },
     {
-      "completion_length": 192.875,
+      "completion_length": 69.3125,
       "epoch": 0.018666666666666668,
-      "grad_norm": 0.0010999701917171478,
-      "kl": 0.0008370049763470888,
+      "grad_norm": 11.347992897033691,
+      "kl": 0.015205658972263336,
       "learning_rate": 4.762067631165049e-07,
       "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "reward": 0.0625,
+      "reward_std": 0.125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.0625,
       "step": 14
     },
     {
-      "completion_length": 205.6875,
+      "completion_length": 91.6875,
       "epoch": 0.02,
-      "grad_norm": 0.0007266216562129557,
-      "kl": 0.0007688422920182347,
+      "grad_norm": 6.206334114074707,
+      "kl": 0.0514555498957634,
       "learning_rate": 4.7275163104709194e-07,
-      "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "loss": 0.0001,
+      "reward": 0.0625,
+      "reward_std": 0.125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.0625,
       "step": 15
     },
     {
-      "completion_length": 163.0625,
+      "completion_length": 123.125,
       "epoch": 0.021333333333333333,
-      "grad_norm": 0.002079131081700325,
-      "kl": 0.0009201083448715508,
+      "grad_norm": 9.800585746765137,
+      "kl": 0.0126947071403265,
       "learning_rate": 4.6907667001096585e-07,
       "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "reward": 0.125,
+      "reward_std": 0.25,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.125,
       "step": 16
     },
     {
-      "completion_length": 131.6875,
+      "completion_length": 110.625,
       "epoch": 0.02266666666666667,
-      "grad_norm": 0.005897476337850094,
-      "kl": 0.0009303554543294013,
+      "grad_norm": 8.219995498657227,
+      "kl": 0.037900131195783615,
       "learning_rate": 4.6518550675098587e-07,
       "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "reward": 0.125,
+      "reward_std": 0.25,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.125,
       "step": 17
     },
     {
-      "completion_length": 151.4375,
+      "completion_length": 102.5625,
       "epoch": 0.024,
-      "grad_norm": 0.002442071447148919,
-      "kl": 0.0010876132873818278,
+      "grad_norm": 5.030233383178711,
+      "kl": 0.015008080750703812,
       "learning_rate": 4.6108198137550377e-07,
       "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "reward": 0.125,
+      "reward_std": 0.14433756470680237,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.125,
       "step": 18
     },
     {
-      "completion_length": 177.9375,
+      "completion_length": 88.25,
       "epoch": 0.025333333333333333,
-      "grad_norm": 9.569433212280273,
-      "kl": 0.004583852831274271,
+      "grad_norm": 3.730710029602051,
+      "kl": 0.04922018200159073,
       "learning_rate": 4.567701435686404e-07,
       "loss": 0.0,
-      "reward": 0.0625,
-      "reward_std": 0.125,
+      "reward": 0.03125,
+      "reward_std": 0.0625,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0625,
+      "rewards/format_reward_func": 0.03125,
       "step": 19
     },
     {
-      "completion_length": 176.8125,
+      "completion_length": 120.0625,
       "epoch": 0.02666666666666667,
-      "grad_norm": 0.0038311942480504513,
-      "kl": 0.001420724904164672,
+      "grad_norm": 8.545513153076172,
+      "kl": 0.0323660746216774,
       "learning_rate": 4.5225424859373684e-07,
       "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "reward": 0.25,
+      "reward_std": 0.39433756470680237,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.25,
       "step": 20
     },
     {
-      "completion_length": 145.6875,
+      "completion_length": 133.875,
       "epoch": 0.028,
-      "grad_norm": 0.019769007340073586,
-      "kl": 0.003168366150930524,
+      "grad_norm": 7.563712120056152,
+      "kl": 0.032230477780103683,
       "learning_rate": 4.475387530939226e-07,
       "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "reward": 0.25,
+      "reward_std": 0.25,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.25,
       "step": 21
     },
     {
-      "completion_length": 99.125,
+      "completion_length": 109.4375,
       "epoch": 0.029333333333333333,
-      "grad_norm": 0.019412707537412643,
-      "kl": 0.0031863912008702755,
+      "grad_norm": 13.283954620361328,
+      "kl": 0.036050185561180115,
       "learning_rate": 4.426283106939473e-07,
       "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "reward": 0.3125,
+      "reward_std": 0.41367512941360474,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.3125,
       "step": 22
     },
     {
-      "completion_length": 166.4375,
+      "completion_length": 139.0,
       "epoch": 0.030666666666666665,
-      "grad_norm": 3.5533502101898193,
-      "kl": 0.002899652114138007,
+      "grad_norm": 8.06175422668457,
+      "kl": 0.01777772419154644,
       "learning_rate": 4.375277674076149e-07,
       "loss": 0.0,
-      "reward": 0.0625,
-      "reward_std": 0.125,
+      "reward": 0.3125,
+      "reward_std": 0.41367512941360474,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0625,
+      "rewards/format_reward_func": 0.3125,
       "step": 23
     },
     {
-      "completion_length": 138.3125,
+      "completion_length": 130.5,
       "epoch": 0.032,
-      "grad_norm": 0.006241905968636274,
-      "kl": 0.0016803923062980175,
+      "grad_norm": 10.465758323669434,
+      "kl": 0.028637699782848358,
       "learning_rate": 4.3224215685535287e-07,
       "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "reward": 0.46875,
+      "reward_std": 0.3696783781051636,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.46875,
       "step": 24
     },
     {
-      "completion_length": 158.4375,
+      "completion_length": 180.6875,
       "epoch": 0.03333333333333333,
-      "grad_norm": 4.496654987335205,
-      "kl": 0.001099013490602374,
+      "grad_norm": 9.501301765441895,
+      "kl": 0.02521451562643051,
       "learning_rate": 4.2677669529663686e-07,
       "loss": 0.0,
-      "reward": 0.0625,
-      "reward_std": 0.125,
+      "reward": 0.3125,
+      "reward_std": 0.51933753490448,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0625,
+      "rewards/format_reward_func": 0.3125,
       "step": 25
     },
     {
-      "completion_length": 189.25,
+      "completion_length": 158.6875,
       "epoch": 0.034666666666666665,
-      "grad_norm": 0.004112588707357645,
-      "kl": 0.0016956878826022148,
+      "grad_norm": 10.960349082946777,
+      "kl": 0.0829106867313385,
       "learning_rate": 4.2113677648217216e-07,
-      "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "loss": 0.0001,
+      "reward": 0.25,
+      "reward_std": 0.28867512941360474,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.25,
       "step": 26
     },
     {
-      "completion_length": 188.1875,
+      "completion_length": 138.25,
       "epoch": 0.036,
-      "grad_norm": 3.3644537925720215,
-      "kl": 0.0010217109229415655,
+      "grad_norm": 7.017608165740967,
+      "kl": 0.02953716553747654,
       "learning_rate": 4.1532796633091294e-07,
       "loss": 0.0,
-      "reward": 0.0625,
-      "reward_std": 0.125,
+      "reward": 0.5,
+      "reward_std": 0.39433756470680237,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0625,
+      "rewards/format_reward_func": 0.5,
       "step": 27
     },
     {
-      "completion_length": 185.1875,
+      "completion_length": 112.125,
       "epoch": 0.037333333333333336,
-      "grad_norm": 0.0016409006202593446,
-      "kl": 0.0010863029165193439,
+      "grad_norm": 9.538881301879883,
+      "kl": 0.06156347692012787,
       "learning_rate": 4.0935599743717244e-07,
-      "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "loss": 0.0001,
+      "reward": 0.625,
+      "reward_std": 0.5,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.625,
       "step": 28
     },
     {
-      "completion_length": 144.3125,
+      "completion_length": 151.25,
       "epoch": 0.03866666666666667,
-      "grad_norm": 0.011104789562523365,
-      "kl": 0.0030689446721225977,
+      "grad_norm": 10.00710678100586,
+      "kl": 0.027312466874718666,
       "learning_rate": 4.0322676341324414e-07,
       "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "reward": 0.4375,
+      "reward_std": 0.51933753490448,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.4375,
       "step": 29
     },
     {
-      "completion_length": 221.75,
+      "completion_length": 160.1875,
       "epoch": 0.04,
-      "grad_norm": 3.384615659713745,
-      "kl": 0.0014688875526189804,
+      "grad_norm": 10.74777603149414,
+      "kl": 0.04307672381401062,
       "learning_rate": 3.9694631307311825e-07,
       "loss": 0.0,
-      "reward": 0.0625,
-      "reward_std": 0.125,
+      "reward": 0.25,
+      "reward_std": 0.5,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0625,
+      "rewards/format_reward_func": 0.25,
       "step": 30
     },
     {
-      "completion_length": 178.25,
+      "completion_length": 141.0625,
       "epoch": 0.04133333333333333,
-      "grad_norm": 0.007718691602349281,
-      "kl": 0.002737755421549082,
+      "grad_norm": 11.85556697845459,
+      "kl": 0.05974256619811058,
       "learning_rate": 3.9052084446303265e-07,
-      "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "loss": 0.0001,
+      "reward": 0.40625,
+      "reward_std": 0.4946783781051636,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.40625,
       "step": 31
     },
     {
-      "completion_length": 165.1875,
+      "completion_length": 153.875,
       "epoch": 0.042666666666666665,
-      "grad_norm": 3.6017088890075684,
-      "kl": 0.0028503944631665945,
+      "grad_norm": 8.729798316955566,
+      "kl": 0.042332496494054794,
       "learning_rate": 3.839566987447491e-07,
       "loss": 0.0,
-      "reward": 0.0625,
-      "reward_std": 0.125,
+      "reward": 0.4375,
+      "reward_std": 0.5580127239227295,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0625,
+      "rewards/format_reward_func": 0.4375,
       "step": 32
     },
     {
-      "completion_length": 144.0,
+      "completion_length": 118.25,
       "epoch": 0.044,
-      "grad_norm": 4.880671977996826,
-      "kl": 0.009817617945373058,
+      "grad_norm": 6.3447585105896,
+      "kl": 0.07207943499088287,
       "learning_rate": 3.7726035393759283e-07,
-      "loss": 0.0,
-      "reward": 0.0625,
-      "reward_std": 0.125,
+      "loss": 0.0001,
+      "reward": 0.8125,
+      "reward_std": 0.26933756470680237,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0625,
+      "rewards/format_reward_func": 0.8125,
       "step": 33
     },
     {
-      "completion_length": 129.4375,
+      "completion_length": 112.5,
       "epoch": 0.04533333333333334,
-      "grad_norm": 0.002933287527412176,
-      "kl": 0.0015820721164345741,
+      "grad_norm": 6.1037797927856445,
+      "kl": 0.09316730499267578,
       "learning_rate": 3.704384185254288e-07,
-      "loss": 0.0,
-      "reward": 0.0,
-      "reward_std": 0.0,
+      "loss": 0.0001,
+      "reward": 0.8125,
+      "reward_std": 0.26933756470680237,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0,
+      "rewards/format_reward_func": 0.8125,
       "step": 34
     },
     {
-      "completion_length": 102.9375,
+      "completion_length": 139.8125,
       "epoch": 0.04666666666666667,
-      "grad_norm": 8.057755470275879,
-      "kl": 0.011732880026102066,
+      "grad_norm": 7.664778232574463,
+      "kl": 0.0723225474357605,
       "learning_rate": 3.634976249348867e-07,
-      "loss": 0.0,
-      "reward": 0.0625,
-      "reward_std": 0.125,
+      "loss": 0.0001,
+      "reward": 0.78125,
+      "reward_std": 0.3696783781051636,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0625,
+      "rewards/format_reward_func": 0.78125,
       "step": 35
     },
     {
-      "completion_length": 144.0,
+      "completion_length": 112.375,
       "epoch": 0.048,
-      "grad_norm": 3.886326551437378,
-      "kl": 0.0062111858278512955,
+      "grad_norm": 9.651317596435547,
+      "kl": 0.09862169623374939,
       "learning_rate": 3.5644482289126813e-07,
-      "loss": 0.0,
-      "reward": 0.0625,
-      "reward_std": 0.125,
+      "loss": 0.0001,
+      "reward": 0.5625,
+      "reward_std": 0.51933753490448,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0625,
+      "rewards/format_reward_func": 0.5625,
       "step": 36
     },
     {
-      "completion_length": 138.25,
+      "completion_length": 108.9375,
       "epoch": 0.04933333333333333,
-      "grad_norm": 8.39665412902832,
-      "kl": 0.0049956561997532845,
+      "grad_norm": 8.387043952941895,
+      "kl": 0.10006917268037796,
       "learning_rate": 3.492869726586951e-07,
-      "loss": 0.0,
-      "reward": 0.125,
-      "reward_std": 0.25,
+      "loss": 0.0001,
+      "reward": 0.8125,
+      "reward_std": 0.375,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.125,
+      "rewards/format_reward_func": 0.8125,
       "step": 37
     },
     {
-      "completion_length": 184.1875,
+      "completion_length": 122.0,
       "epoch": 0.050666666666666665,
-      "grad_norm": 3.575878858566284,
-      "kl": 0.009746416471898556,
+      "grad_norm": 8.030802726745605,
+      "kl": 0.13344745337963104,
       "learning_rate": 3.4203113817116953e-07,
-      "loss": 0.0,
-      "reward": 0.0625,
-      "reward_std": 0.125,
+      "loss": 0.0001,
+      "reward": 0.8125,
+      "reward_std": 0.375,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0625,
+      "rewards/format_reward_func": 0.8125,
       "step": 38
     },
     {
-      "completion_length": 173.1875,
+      "completion_length": 171.8125,
       "epoch": 0.052,
-      "grad_norm": 4.618286609649658,
-      "kl": 0.004162437282502651,
+      "grad_norm": 68.77984619140625,
+      "kl": 0.05845522880554199,
       "learning_rate": 3.346844800613229e-07,
-      "loss": 0.0,
-      "reward": 0.0625,
-      "reward_std": 0.125,
+      "loss": 0.0001,
+      "reward": 0.6875,
+      "reward_std": 0.48935678601264954,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0625,
+      "rewards/format_reward_func": 0.6875,
       "step": 39
     },
     {
-      "completion_length": 144.75,
+      "completion_length": 109.4375,
       "epoch": 0.05333333333333334,
-      "grad_norm": 4.964860916137695,
-      "kl": 0.0039725033566355705,
+      "grad_norm": 7.004795551300049,
+      "kl": 0.07453721761703491,
       "learning_rate": 3.272542485937368e-07,
-      "loss": 0.0,
-      "reward": 0.0625,
-      "reward_std": 0.125,
+      "loss": 0.0001,
+      "reward": 0.90625,
+      "reward_std": 0.1875,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0625,
+      "rewards/format_reward_func": 0.90625,
       "step": 40
     },
     {
-      "completion_length": 191.5,
+      "completion_length": 129.75,
       "epoch": 0.05466666666666667,
-      "grad_norm": 5.967226028442383,
-      "kl": 0.008939428254961967,
+      "grad_norm": 7.910297393798828,
+      "kl": 0.09665161371231079,
       "learning_rate": 3.1974777650980734e-07,
-      "loss": 0.0,
-      "reward": 0.125,
-      "reward_std": 0.25,
+      "loss": 0.0001,
+      "reward": 0.75,
+      "reward_std": 0.36435678601264954,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.125,
+      "rewards/format_reward_func": 0.75,
       "step": 41
     },
     {
-      "completion_length": 149.6875,
+      "completion_length": 138.6875,
       "epoch": 0.056,
-      "grad_norm": 5.210999488830566,
-      "kl": 0.01842350885272026,
+      "grad_norm": 6.366047382354736,
+      "kl": 0.06816712021827698,
       "learning_rate": 3.121724717912138e-07,
-      "loss": 0.0,
-      "reward": 0.0625,
-      "reward_std": 0.125,
+      "loss": 0.0001,
+      "reward": 0.875,
+      "reward_std": 0.18217839300632477,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0625,
+      "rewards/format_reward_func": 0.875,
       "step": 42
     },
     {
-      "completion_length": 138.5,
+      "completion_length": 163.4375,
       "epoch": 0.05733333333333333,
-      "grad_norm": 12.056984901428223,
-      "kl": 0.019806817173957825,
+      "grad_norm": 9.733158111572266,
+      "kl": 0.07456796616315842,
       "learning_rate": 3.0453581034913565e-07,
-      "loss": 0.0,
-      "reward": 0.3125,
-      "reward_std": 0.51933753490448,
+      "loss": 0.0001,
+      "reward": 0.75,
+      "reward_std": 0.39433756470680237,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.3125,
+      "rewards/format_reward_func": 0.75,
       "step": 43
     },
     {
-      "completion_length": 115.9375,
+      "completion_length": 149.375,
       "epoch": 0.058666666666666666,
-      "grad_norm": 17.690818786621094,
-      "kl": 0.054062891751527786,
+      "grad_norm": 7.283127784729004,
+      "kl": 0.10417325794696808,
       "learning_rate": 2.968453286464312e-07,
       "loss": 0.0001,
-      "reward": 0.1875,
+      "reward": 0.8125,
       "reward_std": 0.375,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.1875,
+      "rewards/format_reward_func": 0.8125,
       "step": 44
     },
     {
-      "completion_length": 137.1875,
+      "completion_length": 151.8125,
       "epoch": 0.06,
-      "grad_norm": 12.18422794342041,
-      "kl": 0.025158584117889404,
+      "grad_norm": 8.65285873413086,
+      "kl": 0.08351579308509827,
       "learning_rate": 2.8910861626005773e-07,
-      "loss": 0.0,
-      "reward": 0.25,
-      "reward_std": 0.5,
+      "loss": 0.0001,
+      "reward": 0.71875,
+      "reward_std": 0.45683756470680237,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.25,
+      "rewards/format_reward_func": 0.71875,
       "step": 45
     },
     {
-      "completion_length": 145.0625,
+      "completion_length": 122.1875,
       "epoch": 0.06133333333333333,
-      "grad_norm": 4.150605201721191,
-      "kl": 0.011470025405287743,
+      "grad_norm": 6.481723308563232,
+      "kl": 0.0788784921169281,
       "learning_rate": 2.8133330839107604e-07,
-      "loss": 0.0,
-      "reward": 0.0625,
-      "reward_std": 0.125,
+      "loss": 0.0001,
+      "reward": 0.84375,
+      "reward_std": 0.24467839300632477,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.0625,
+      "rewards/format_reward_func": 0.84375,
       "step": 46
     },
     {
-      "completion_length": 149.6875,
+      "completion_length": 177.125,
       "epoch": 0.06266666666666666,
-      "grad_norm": 7.5838541984558105,
-      "kl": 0.028349004685878754,
+      "grad_norm": 6.107300281524658,
+      "kl": 0.10843782126903534,
       "learning_rate": 2.735270783296286e-07,
-      "loss": 0.0,
-      "reward": 0.3125,
-      "reward_std": 0.41367512941360474,
+      "loss": 0.0001,
+      "reward": 0.84375,
+      "reward_std": 0.24467839300632477,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.3125,
+      "rewards/format_reward_func": 0.84375,
       "step": 47
     },
     {
-      "completion_length": 129.6875,
+      "completion_length": 115.875,
       "epoch": 0.064,
-      "grad_norm": 7.82366943359375,
-      "kl": 0.012228617444634438,
+      "grad_norm": 5.656793117523193,
+      "kl": 0.11855285614728928,
       "learning_rate": 2.6569762988232837e-07,
-      "loss": 0.0,
-      "reward": 0.1875,
-      "reward_std": 0.375,
+      "loss": 0.0001,
+      "reward": 0.90625,
+      "reward_std": 0.1875,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.1875,
+      "rewards/format_reward_func": 0.90625,
       "step": 48
     },
     {
-      "completion_length": 167.125,
+      "completion_length": 137.6875,
       "epoch": 0.06533333333333333,
-      "grad_norm": 6.554472923278809,
-      "kl": 0.01310904324054718,
+      "grad_norm": 7.453767776489258,
+      "kl": 0.10749398171901703,
       "learning_rate": 2.5785268976953204e-07,
-      "loss": 0.0,
-      "reward": 0.1875,
-      "reward_std": 0.375,
+      "loss": 0.0001,
+      "reward": 0.90625,
+      "reward_std": 0.1875,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.1875,
+      "rewards/format_reward_func": 0.90625,
       "step": 49
     },
     {
-      "completion_length": 157.1875,
+      "completion_length": 117.375,
       "epoch": 0.06666666666666667,
-      "grad_norm": 6.399176120758057,
-      "kl": 0.02061685547232628,
+      "grad_norm": 6.2615861892700195,
+      "kl": 0.09339688718318939,
       "learning_rate": 2.5e-07,
-      "loss": 0.0,
-      "reward": 0.4375,
-      "reward_std": 0.41367512941360474,
+      "loss": 0.0001,
+      "reward": 0.8125,
+      "reward_std": 0.26933756470680237,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.4375,
+      "rewards/format_reward_func": 0.8125,
       "step": 50
     },
     {
-      "completion_length": 164.875,
+      "completion_length": 120.625,
       "epoch": 0.068,
-      "grad_norm": 7.1559977531433105,
-      "kl": 0.02672770991921425,
+      "grad_norm": 7.3459577560424805,
+      "kl": 0.08987420052289963,
       "learning_rate": 2.4214731023046794e-07,
-      "loss": 0.0,
-      "reward": 0.25,
-      "reward_std": 0.39433756470680237,
+      "loss": 0.0001,
+      "reward": 0.84375,
+      "reward_std": 0.3125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.25,
+      "rewards/format_reward_func": 0.84375,
       "step": 51
     },
     {
-      "completion_length": 131.75,
+      "completion_length": 130.4375,
       "epoch": 0.06933333333333333,
-      "grad_norm": 10.416351318359375,
-      "kl": 0.09771859645843506,
+      "grad_norm": 7.253162384033203,
+      "kl": 0.0824127197265625,
       "learning_rate": 2.3430237011767164e-07,
       "loss": 0.0001,
-      "reward": 0.6875,
-      "reward_std": 0.41367512941360474,
+      "reward": 0.8125,
+      "reward_std": 0.26933756470680237,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.6875,
+      "rewards/format_reward_func": 0.8125,
       "step": 52
     },
     {
-      "completion_length": 134.4375,
+      "completion_length": 102.875,
       "epoch": 0.07066666666666667,
-      "grad_norm": 9.832062721252441,
-      "kl": 0.03194844722747803,
+      "grad_norm": 7.457027435302734,
+      "kl": 0.08334946632385254,
       "learning_rate": 2.264729216703714e-07,
-      "loss": 0.0,
-      "reward": 0.375,
-      "reward_std": 0.5386751294136047,
+      "loss": 0.0001,
+      "reward": 0.875,
+      "reward_std": 0.25,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.375,
+      "rewards/format_reward_func": 0.875,
       "step": 53
     },
     {
-      "completion_length": 155.375,
+      "completion_length": 127.75,
       "epoch": 0.072,
-      "grad_norm": 7.258142471313477,
-      "kl": 0.022360337898135185,
+      "grad_norm": 0.01185312308371067,
+      "kl": 0.094039186835289,
       "learning_rate": 2.1866669160892389e-07,
-      "loss": 0.0,
-      "reward": 0.5,
-      "reward_std": 0.39433756470680237,
+      "loss": 0.0001,
+      "reward": 1.0,
+      "reward_std": 0.0,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.5,
+      "rewards/format_reward_func": 1.0,
       "step": 54
     },
     {
-      "completion_length": 128.375,
+      "completion_length": 86.5,
       "epoch": 0.07333333333333333,
-      "grad_norm": 8.793972969055176,
-      "kl": 0.029077202081680298,
+      "grad_norm": 11.868236541748047,
+      "kl": 0.22406581044197083,
       "learning_rate": 2.1089138373994222e-07,
-      "loss": 0.0,
-      "reward": 0.4375,
-      "reward_std": 0.5580127239227295,
+      "loss": 0.0002,
+      "reward": 0.84375,
+      "reward_std": 0.3125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.4375,
+      "rewards/format_reward_func": 0.84375,
       "step": 55
     },
     {
-      "completion_length": 127.9375,
+      "completion_length": 105.0625,
       "epoch": 0.07466666666666667,
-      "grad_norm": 22.435115814208984,
-      "kl": 0.23208478093147278,
+      "grad_norm": 3.1902968883514404,
+      "kl": 0.10381343960762024,
       "learning_rate": 2.0315467135356878e-07,
-      "loss": 0.0002,
-      "reward": 0.5,
-      "reward_std": 0.5386751294136047,
+      "loss": 0.0001,
+      "reward": 0.96875,
+      "reward_std": 0.0625,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.5,
+      "rewards/format_reward_func": 0.96875,
       "step": 56
     },
     {
-      "completion_length": 115.6875,
+      "completion_length": 77.8125,
       "epoch": 0.076,
-      "grad_norm": 7.384529113769531,
-      "kl": 0.02935698628425598,
+      "grad_norm": 0.04087565839290619,
+      "kl": 0.17592526972293854,
       "learning_rate": 1.954641896508644e-07,
-      "loss": 0.0,
-      "reward": 0.5625,
-      "reward_std": 0.41367512941360474,
+      "loss": 0.0002,
+      "reward": 1.0,
+      "reward_std": 0.0,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.5625,
+      "rewards/format_reward_func": 1.0,
       "step": 57
     },
     {
-      "completion_length": 161.4375,
+      "completion_length": 114.5,
       "epoch": 0.07733333333333334,
-      "grad_norm": 5.652832984924316,
-      "kl": 0.02788429707288742,
+      "grad_norm": 4.14316463470459,
+      "kl": 0.09431131184101105,
       "learning_rate": 1.8782752820878633e-07,
-      "loss": 0.0,
-      "reward": 0.5625,
-      "reward_std": 0.26933756470680237,
+      "loss": 0.0001,
+      "reward": 0.875,
+      "reward_std": 0.14433756470680237,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.5625,
+      "rewards/format_reward_func": 0.875,
       "step": 58
     },
     {
-      "completion_length": 114.0625,
+      "completion_length": 99.25,
       "epoch": 0.07866666666666666,
-      "grad_norm": 7.544243335723877,
-      "kl": 0.0415709987282753,
+      "grad_norm": 12.474014282226562,
+      "kl": 0.16972073912620544,
       "learning_rate": 1.802522234901927e-07,
-      "loss": 0.0,
-      "reward": 0.75,
-      "reward_std": 0.25,
+      "loss": 0.0002,
+      "reward": 0.6875,
+      "reward_std": 0.41367512941360474,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.75,
+      "rewards/format_reward_func": 0.6875,
       "step": 59
     },
     {
-      "completion_length": 128.25,
+      "completion_length": 114.6875,
       "epoch": 0.08,
-      "grad_norm": 6.015617847442627,
-      "kl": 0.038651518523693085,
+      "grad_norm": 0.014834249392151833,
+      "kl": 0.11283782124519348,
       "learning_rate": 1.7274575140626315e-07,
-      "loss": 0.0,
-      "reward": 0.875,
-      "reward_std": 0.25,
+      "loss": 0.0001,
+      "reward": 1.0,
+      "reward_std": 0.0,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.875,
+      "rewards/format_reward_func": 1.0,
       "step": 60
     },
     {
-      "completion_length": 146.125,
+      "completion_length": 107.6875,
       "epoch": 0.08133333333333333,
-      "grad_norm": 7.534911155700684,
-      "kl": 0.023068124428391457,
+      "grad_norm": 11.97496509552002,
+      "kl": 0.2880978286266327,
       "learning_rate": 1.6531551993867715e-07,
-      "loss": 0.0,
-      "reward": 0.6875,
-      "reward_std": 0.375,
+      "loss": 0.0003,
+      "reward": 0.84375,
+      "reward_std": 0.3125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.6875,
+      "rewards/format_reward_func": 0.84375,
       "step": 61
     },
     {
-      "completion_length": 136.6875,
+      "completion_length": 115.3125,
       "epoch": 0.08266666666666667,
-      "grad_norm": 7.638873100280762,
-      "kl": 0.03138712793588638,
+      "grad_norm": 0.06830989569425583,
+      "kl": 0.12032976001501083,
       "learning_rate": 1.579688618288305e-07,
-      "loss": 0.0,
-      "reward": 0.75,
-      "reward_std": 0.39433756470680237,
+      "loss": 0.0001,
+      "reward": 1.0,
+      "reward_std": 0.0,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.75,
+      "rewards/format_reward_func": 1.0,
       "step": 62
     },
     {
-      "completion_length": 124.8125,
+      "completion_length": 109.9375,
       "epoch": 0.084,
-      "grad_norm": 22.380794525146484,
-      "kl": 0.28954756259918213,
+      "grad_norm": 12.59013557434082,
+      "kl": 0.16462820768356323,
       "learning_rate": 1.5071302734130486e-07,
-      "loss": 0.0003,
-      "reward": 0.75,
-      "reward_std": 0.5,
+      "loss": 0.0002,
+      "reward": 0.875,
+      "reward_std": 0.25,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.75,
+      "rewards/format_reward_func": 0.875,
       "step": 63
     },
     {
-      "completion_length": 126.3125,
+      "completion_length": 112.6875,
       "epoch": 0.08533333333333333,
-      "grad_norm": 7.005850791931152,
-      "kl": 0.03995266556739807,
+      "grad_norm": 0.011938858777284622,
+      "kl": 0.105996273458004,
       "learning_rate": 1.4355517710873182e-07,
-      "loss": 0.0,
-      "reward": 0.8125,
-      "reward_std": 0.375,
+      "loss": 0.0001,
+      "reward": 1.0,
+      "reward_std": 0.0,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.8125,
+      "rewards/format_reward_func": 1.0,
       "step": 64
     },
     {
-      "completion_length": 167.5,
+      "completion_length": 113.3125,
       "epoch": 0.08666666666666667,
-      "grad_norm": 6.591019153594971,
-      "kl": 0.029363617300987244,
+      "grad_norm": 3.5281906127929688,
+      "kl": 0.11741024255752563,
       "learning_rate": 1.365023750651133e-07,
-      "loss": 0.0,
-      "reward": 0.8125,
-      "reward_std": 0.375,
+      "loss": 0.0001,
+      "reward": 0.96875,
+      "reward_std": 0.0625,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.8125,
+      "rewards/format_reward_func": 0.96875,
       "step": 65
     },
     {
-      "completion_length": 120.0,
+      "completion_length": 115.375,
       "epoch": 0.088,
-      "grad_norm": 3.859731674194336,
-      "kl": 0.05216670036315918,
+      "grad_norm": 5.508726596832275,
+      "kl": 0.11176759004592896,
       "learning_rate": 1.2956158147457114e-07,
       "loss": 0.0001,
       "reward": 0.9375,
@@ -867,207 +867,207 @@
       "step": 66
     },
     {
-      "completion_length": 116.0,
+      "completion_length": 118.75,
       "epoch": 0.08933333333333333,
-      "grad_norm": 11.705424308776855,
-      "kl": 0.42816463112831116,
+      "grad_norm": 6.57678747177124,
+      "kl": 0.12585385143756866,
       "learning_rate": 1.2273964606240718e-07,
-      "loss": 0.0004,
-      "reward": 0.625,
-      "reward_std": 0.4330126941204071,
+      "loss": 0.0001,
+      "reward": 0.90625,
+      "reward_std": 0.1875,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.625,
+      "rewards/format_reward_func": 0.90625,
       "step": 67
     },
     {
-      "completion_length": 157.125,
+      "completion_length": 128.8125,
       "epoch": 0.09066666666666667,
-      "grad_norm": 7.195036888122559,
-      "kl": 0.030014928430318832,
+      "grad_norm": 0.012199473567306995,
+      "kl": 0.1194550171494484,
       "learning_rate": 1.1604330125525078e-07,
-      "loss": 0.0,
-      "reward": 0.6875,
-      "reward_std": 0.41367512941360474,
+      "loss": 0.0001,
+      "reward": 1.0,
+      "reward_std": 0.0,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.6875,
+      "rewards/format_reward_func": 1.0,
       "step": 68
     },
     {
-      "completion_length": 114.5,
+      "completion_length": 102.75,
       "epoch": 0.092,
-      "grad_norm": 6.199704647064209,
-      "kl": 0.04237810894846916,
+      "grad_norm": 6.777124881744385,
+      "kl": 0.14668866991996765,
       "learning_rate": 1.0947915553696741e-07,
-      "loss": 0.0,
-      "reward": 0.8125,
-      "reward_std": 0.26933756470680237,
+      "loss": 0.0001,
+      "reward": 0.9375,
+      "reward_std": 0.125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.8125,
+      "rewards/format_reward_func": 0.9375,
       "step": 69
     },
     {
-      "completion_length": 143.125,
+      "completion_length": 96.25,
       "epoch": 0.09333333333333334,
-      "grad_norm": 8.729787826538086,
-      "kl": 0.0419083908200264,
+      "grad_norm": 6.230812072753906,
+      "kl": 0.1095753163099289,
       "learning_rate": 1.0305368692688174e-07,
-      "loss": 0.0,
-      "reward": 0.625,
-      "reward_std": 0.5386751294136047,
+      "loss": 0.0001,
+      "reward": 0.90625,
+      "reward_std": 0.1875,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.625,
+      "rewards/format_reward_func": 0.90625,
       "step": 70
     },
     {
-      "completion_length": 142.0,
+      "completion_length": 98.25,
       "epoch": 0.09466666666666666,
-      "grad_norm": 7.905228137969971,
-      "kl": 0.04787478223443031,
+      "grad_norm": 3.248274326324463,
+      "kl": 0.1126992404460907,
       "learning_rate": 9.677323658675593e-08,
-      "loss": 0.0,
-      "reward": 0.625,
-      "reward_std": 0.5,
+      "loss": 0.0001,
+      "reward": 0.9375,
+      "reward_std": 0.125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.625,
+      "rewards/format_reward_func": 0.9375,
       "step": 71
     },
     {
-      "completion_length": 144.75,
+      "completion_length": 112.8125,
       "epoch": 0.096,
-      "grad_norm": 6.076593399047852,
-      "kl": 0.03787325695157051,
+      "grad_norm": 5.262601852416992,
+      "kl": 0.1305035650730133,
       "learning_rate": 9.064400256282755e-08,
-      "loss": 0.0,
-      "reward": 0.5625,
-      "reward_std": 0.26933756470680237,
+      "loss": 0.0001,
+      "reward": 0.9375,
+      "reward_std": 0.125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.5625,
+      "rewards/format_reward_func": 0.9375,
       "step": 72
     },
     {
-      "completion_length": 170.625,
+      "completion_length": 121.375,
       "epoch": 0.09733333333333333,
-      "grad_norm": 5.793034076690674,
-      "kl": 0.025196196511387825,
+      "grad_norm": 0.01645738258957863,
+      "kl": 0.12455137819051743,
       "learning_rate": 8.467203366908707e-08,
-      "loss": 0.0,
-      "reward": 0.8125,
-      "reward_std": 0.375,
+      "loss": 0.0001,
+      "reward": 1.0,
+      "reward_std": 0.0,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.8125,
+      "rewards/format_reward_func": 1.0,
       "step": 73
     },
     {
-      "completion_length": 128.25,
+      "completion_length": 113.625,
       "epoch": 0.09866666666666667,
-      "grad_norm": 7.563605308532715,
-      "kl": 0.04877077788114548,
+      "grad_norm": 5.738455295562744,
+      "kl": 0.14311644434928894,
       "learning_rate": 7.886322351782782e-08,
-      "loss": 0.0,
-      "reward": 0.75,
-      "reward_std": 0.39433756470680237,
+      "loss": 0.0001,
+      "reward": 0.875,
+      "reward_std": 0.25,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.75,
+      "rewards/format_reward_func": 0.875,
       "step": 74
     },
     {
-      "completion_length": 104.3125,
+      "completion_length": 104.375,
       "epoch": 0.1,
-      "grad_norm": 9.703227043151855,
-      "kl": 0.0765969455242157,
+      "grad_norm": 0.014866613782942295,
+      "kl": 0.13841284811496735,
       "learning_rate": 7.322330470336313e-08,
       "loss": 0.0001,
-      "reward": 0.5625,
-      "reward_std": 0.41367512941360474,
+      "reward": 1.0,
+      "reward_std": 0.0,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.5625,
+      "rewards/format_reward_func": 1.0,
       "step": 75
     },
     {
-      "completion_length": 131.5,
+      "completion_length": 96.125,
       "epoch": 0.10133333333333333,
-      "grad_norm": 8.99862003326416,
-      "kl": 0.04248180240392685,
+      "grad_norm": 4.531452178955078,
+      "kl": 0.11935670673847198,
       "learning_rate": 6.775784314464716e-08,
-      "loss": 0.0,
-      "reward": 0.75,
-      "reward_std": 0.5,
+      "loss": 0.0001,
+      "reward": 0.9375,
+      "reward_std": 0.125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.75,
+      "rewards/format_reward_func": 0.9375,
       "step": 76
     },
     {
-      "completion_length": 140.625,
+      "completion_length": 102.0,
       "epoch": 0.10266666666666667,
-      "grad_norm": 8.795363426208496,
-      "kl": 0.043973423540592194,
+      "grad_norm": 0.017895404249429703,
+      "kl": 0.15568867325782776,
       "learning_rate": 6.24722325923851e-08,
-      "loss": 0.0,
-      "reward": 0.5625,
-      "reward_std": 0.51933753490448,
+      "loss": 0.0002,
+      "reward": 1.0,
+      "reward_std": 0.0,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.5625,
+      "rewards/format_reward_func": 1.0,
       "step": 77
     },
     {
-      "completion_length": 104.5625,
+      "completion_length": 85.25,
       "epoch": 0.104,
-      "grad_norm": 9.773259162902832,
-      "kl": 0.07939579337835312,
+      "grad_norm": 6.45017147064209,
+      "kl": 0.20556744933128357,
       "learning_rate": 5.737168930605271e-08,
-      "loss": 0.0001,
-      "reward": 0.75,
-      "reward_std": 0.25,
+      "loss": 0.0002,
+      "reward": 0.96875,
+      "reward_std": 0.0625,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.75,
+      "rewards/format_reward_func": 0.96875,
       "step": 78
     },
     {
-      "completion_length": 119.1875,
+      "completion_length": 90.25,
       "epoch": 0.10533333333333333,
-      "grad_norm": 6.493321418762207,
-      "kl": 0.07017514854669571,
+      "grad_norm": 7.30122709274292,
+      "kl": 0.1739582121372223,
       "learning_rate": 5.246124690607739e-08,
-      "loss": 0.0001,
-      "reward": 0.75,
-      "reward_std": 0.28867512941360474,
+      "loss": 0.0002,
+      "reward": 0.9375,
+      "reward_std": 0.125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.75,
+      "rewards/format_reward_func": 0.9375,
       "step": 79
     },
     {
-      "completion_length": 121.125,
+      "completion_length": 99.4375,
       "epoch": 0.10666666666666667,
-      "grad_norm": 6.501302719116211,
-      "kl": 0.04043734073638916,
+      "grad_norm": 7.5901103019714355,
+      "kl": 0.1773838996887207,
       "learning_rate": 4.774575140626316e-08,
-      "loss": 0.0,
-      "reward": 0.875,
-      "reward_std": 0.25,
+      "loss": 0.0002,
+      "reward": 0.90625,
+      "reward_std": 0.1875,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.875,
+      "rewards/format_reward_func": 0.90625,
       "step": 80
     },
     {
-      "completion_length": 171.9375,
+      "completion_length": 101.875,
       "epoch": 0.108,
-      "grad_norm": 5.958766937255859,
-      "kl": 0.03187550604343414,
+      "grad_norm": 0.016729678958654404,
+      "kl": 0.16674719750881195,
       "learning_rate": 4.3229856431359513e-08,
-      "loss": 0.0,
-      "reward": 0.75,
-      "reward_std": 0.39433756470680237,
+      "loss": 0.0002,
+      "reward": 1.0,
+      "reward_std": 0.0,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.75,
+      "rewards/format_reward_func": 1.0,
       "step": 81
     },
     {
-      "completion_length": 145.1875,
+      "completion_length": 104.5,
       "epoch": 0.10933333333333334,
-      "grad_norm": 3.5965261459350586,
-      "kl": 0.045581966638565063,
+      "grad_norm": 5.888128757476807,
+      "kl": 0.13316090404987335,
       "learning_rate": 3.8918018624496286e-08,
-      "loss": 0.0,
+      "loss": 0.0001,
       "reward": 0.9375,
       "reward_std": 0.125,
       "rewards/emotion_reward_func": 0.0,
@@ -1075,101 +1075,101 @@
       "step": 82
     },
     {
-      "completion_length": 150.375,
+      "completion_length": 96.5625,
       "epoch": 0.11066666666666666,
-      "grad_norm": 3.3176019191741943,
-      "kl": 0.04733135551214218,
+      "grad_norm": 5.195379734039307,
+      "kl": 0.13178151845932007,
       "learning_rate": 3.481449324901411e-08,
-      "loss": 0.0,
-      "reward": 0.875,
-      "reward_std": 0.14433756470680237,
+      "loss": 0.0001,
+      "reward": 0.96875,
+      "reward_std": 0.0625,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.875,
+      "rewards/format_reward_func": 0.96875,
       "step": 83
     },
     {
-      "completion_length": 125.0,
+      "completion_length": 120.3125,
       "epoch": 0.112,
-      "grad_norm": 5.251501560211182,
-      "kl": 0.04413582757115364,
+      "grad_norm": 0.023672526702284813,
+      "kl": 0.09863791614770889,
       "learning_rate": 3.092332998903416e-08,
-      "loss": 0.0,
-      "reward": 0.875,
-      "reward_std": 0.14433756470680237,
+      "loss": 0.0001,
+      "reward": 1.0,
+      "reward_std": 0.0,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.875,
+      "rewards/format_reward_func": 1.0,
       "step": 84
     },
     {
-      "completion_length": 124.5,
+      "completion_length": 120.3125,
       "epoch": 0.11333333333333333,
-      "grad_norm": 10.067094802856445,
-      "kl": 0.05217205733060837,
+      "grad_norm": 0.016033878549933434,
+      "kl": 0.1298094093799591,
       "learning_rate": 2.724836895290805e-08,
       "loss": 0.0001,
-      "reward": 0.6875,
-      "reward_std": 0.51933753490448,
+      "reward": 1.0,
+      "reward_std": 0.0,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.6875,
+      "rewards/format_reward_func": 1.0,
       "step": 85
     },
     {
-      "completion_length": 129.8125,
+      "completion_length": 123.875,
       "epoch": 0.11466666666666667,
-      "grad_norm": 5.6904096603393555,
-      "kl": 0.039646558463573456,
+      "grad_norm": 0.013259505853056908,
+      "kl": 0.13770553469657898,
       "learning_rate": 2.379323688349516e-08,
-      "loss": 0.0,
-      "reward": 0.875,
-      "reward_std": 0.25,
+      "loss": 0.0001,
+      "reward": 1.0,
+      "reward_std": 0.0,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.875,
+      "rewards/format_reward_func": 1.0,
       "step": 86
     },
     {
-      "completion_length": 114.8125,
+      "completion_length": 93.375,
       "epoch": 0.116,
-      "grad_norm": 3.087869644165039,
-      "kl": 0.050276342779397964,
+      "grad_norm": 7.885369777679443,
+      "kl": 0.14231295883655548,
       "learning_rate": 2.0561343579004715e-08,
       "loss": 0.0001,
-      "reward": 0.9375,
-      "reward_std": 0.125,
+      "reward": 0.875,
+      "reward_std": 0.25,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.9375,
+      "rewards/format_reward_func": 0.875,
       "step": 87
     },
     {
-      "completion_length": 116.875,
+      "completion_length": 106.8125,
       "epoch": 0.11733333333333333,
-      "grad_norm": 13.770880699157715,
-      "kl": 0.655558168888092,
+      "grad_norm": 10.611775398254395,
+      "kl": 0.25752192735671997,
       "learning_rate": 1.7555878527937163e-08,
-      "loss": 0.0007,
-      "reward": 0.75,
-      "reward_std": 0.39433756470680237,
+      "loss": 0.0003,
+      "reward": 0.84375,
+      "reward_std": 0.3125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.75,
+      "rewards/format_reward_func": 0.84375,
       "step": 88
     },
     {
-      "completion_length": 113.375,
+      "completion_length": 105.25,
       "epoch": 0.11866666666666667,
-      "grad_norm": 5.708318710327148,
-      "kl": 0.0568830668926239,
+      "grad_norm": 5.855681419372559,
+      "kl": 0.1377657949924469,
       "learning_rate": 1.4779807761443635e-08,
       "loss": 0.0001,
-      "reward": 0.8125,
-      "reward_std": 0.26933756470680237,
+      "reward": 0.9375,
+      "reward_std": 0.125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.8125,
+      "rewards/format_reward_func": 0.9375,
       "step": 89
     },
     {
-      "completion_length": 144.25,
+      "completion_length": 112.9375,
       "epoch": 0.12,
-      "grad_norm": 5.573944091796875,
-      "kl": 0.05078238993883133,
+      "grad_norm": 7.280832290649414,
+      "kl": 0.13963450491428375,
       "learning_rate": 1.2235870926211616e-08,
       "loss": 0.0001,
       "reward": 0.8125,
@@ -1179,75 +1179,75 @@
       "step": 90
     },
     {
-      "completion_length": 144.3125,
+      "completion_length": 113.0625,
       "epoch": 0.12133333333333333,
-      "grad_norm": 7.064413547515869,
-      "kl": 0.04295295104384422,
+      "grad_norm": 5.792718410491943,
+      "kl": 0.10216458886861801,
       "learning_rate": 9.926578580764234e-09,
-      "loss": 0.0,
-      "reward": 0.75,
-      "reward_std": 0.39433756470680237,
+      "loss": 0.0001,
+      "reward": 0.875,
+      "reward_std": 0.25,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.75,
+      "rewards/format_reward_func": 0.875,
       "step": 91
     },
     {
-      "completion_length": 107.625,
+      "completion_length": 130.625,
       "epoch": 0.12266666666666666,
-      "grad_norm": 10.970053672790527,
-      "kl": 0.04969753324985504,
+      "grad_norm": 3.8016581535339355,
+      "kl": 0.09364865720272064,
       "learning_rate": 7.85420971784223e-09,
-      "loss": 0.0,
-      "reward": 0.75,
-      "reward_std": 0.5,
+      "loss": 0.0001,
+      "reward": 0.9375,
+      "reward_std": 0.125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.75,
+      "rewards/format_reward_func": 0.9375,
       "step": 92
     },
     {
-      "completion_length": 134.9375,
+      "completion_length": 128.8125,
       "epoch": 0.124,
-      "grad_norm": 5.796207427978516,
-      "kl": 0.044127851724624634,
+      "grad_norm": 4.038362979888916,
+      "kl": 0.09943213313817978,
       "learning_rate": 6.020809515313141e-09,
-      "loss": 0.0,
-      "reward": 0.875,
-      "reward_std": 0.25,
+      "loss": 0.0001,
+      "reward": 0.96875,
+      "reward_std": 0.0625,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.875,
+      "rewards/format_reward_func": 0.96875,
       "step": 93
     },
     {
-      "completion_length": 109.75,
+      "completion_length": 81.0,
       "epoch": 0.12533333333333332,
-      "grad_norm": 6.654936790466309,
-      "kl": 0.05904502421617508,
+      "grad_norm": 5.979770660400391,
+      "kl": 0.15632514655590057,
       "learning_rate": 4.4281873178278475e-09,
-      "loss": 0.0001,
-      "reward": 0.875,
-      "reward_std": 0.25,
+      "loss": 0.0002,
+      "reward": 0.9375,
+      "reward_std": 0.125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.875,
+      "rewards/format_reward_func": 0.9375,
       "step": 94
     },
     {
-      "completion_length": 124.125,
+      "completion_length": 95.9375,
       "epoch": 0.12666666666666668,
-      "grad_norm": 8.560564041137695,
-      "kl": 0.6496566534042358,
+      "grad_norm": 8.092153549194336,
+      "kl": 0.16083820164203644,
       "learning_rate": 3.077914851215585e-09,
-      "loss": 0.0006,
-      "reward": 0.5625,
-      "reward_std": 0.375,
+      "loss": 0.0002,
+      "reward": 0.9375,
+      "reward_std": 0.125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.5625,
+      "rewards/format_reward_func": 0.9375,
       "step": 95
     },
     {
-      "completion_length": 131.9375,
+      "completion_length": 109.5,
       "epoch": 0.128,
-      "grad_norm": 5.740026473999023,
-      "kl": 0.06654007732868195,
+      "grad_norm": 7.098442077636719,
+      "kl": 0.11922727525234222,
       "learning_rate": 1.9713246713805587e-09,
       "loss": 0.0001,
       "reward": 0.875,
@@ -1257,55 +1257,55 @@
       "step": 96
     },
     {
-      "completion_length": 104.1875,
+      "completion_length": 104.0,
       "epoch": 0.12933333333333333,
-      "grad_norm": 0.006113509647548199,
-      "kl": 0.06723961234092712,
+      "grad_norm": 6.17384147644043,
+      "kl": 0.23637814819812775,
       "learning_rate": 1.1095088492300008e-09,
-      "loss": 0.0001,
-      "reward": 1.0,
-      "reward_std": 0.0,
+      "loss": 0.0002,
+      "reward": 0.9375,
+      "reward_std": 0.125,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 1.0,
+      "rewards/format_reward_func": 0.9375,
       "step": 97
     },
     {
-      "completion_length": 127.5625,
+      "completion_length": 97.75,
       "epoch": 0.13066666666666665,
-      "grad_norm": 3.994433879852295,
-      "kl": 0.048512913286685944,
+      "grad_norm": 0.04274175688624382,
+      "kl": 0.18250277638435364,
       "learning_rate": 4.933178929321102e-10,
-      "loss": 0.0,
-      "reward": 0.9375,
-      "reward_std": 0.125,
+      "loss": 0.0002,
+      "reward": 1.0,
+      "reward_std": 0.0,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.9375,
+      "rewards/format_reward_func": 1.0,
       "step": 98
     },
     {
-      "completion_length": 150.25,
+      "completion_length": 124.6875,
       "epoch": 0.132,
-      "grad_norm": 5.778770446777344,
-      "kl": 0.03687442094087601,
+      "grad_norm": 0.03310486301779747,
+      "kl": 0.09437094628810883,
       "learning_rate": 1.2335990856709998e-10,
-      "loss": 0.0,
-      "reward": 0.8125,
-      "reward_std": 0.375,
+      "loss": 0.0001,
+      "reward": 1.0,
+      "reward_std": 0.0,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.8125,
+      "rewards/format_reward_func": 1.0,
       "step": 99
     },
     {
-      "completion_length": 127.8125,
+      "completion_length": 116.0,
       "epoch": 0.13333333333333333,
-      "grad_norm": 26.98400115966797,
-      "kl": 0.097214475274086,
+      "grad_norm": 0.0801442489027977,
+      "kl": 0.10857471823692322,
       "learning_rate": 0.0,
       "loss": 0.0001,
-      "reward": 0.9375,
-      "reward_std": 0.125,
+      "reward": 1.0,
+      "reward_std": 0.0,
       "rewards/emotion_reward_func": 0.0,
-      "rewards/format_reward_func": 0.9375,
+      "rewards/format_reward_func": 1.0,
       "step": 100
     }
   ],