Model save

README.md

@@ -3,8 +3,8 @@ library_name: transformers
 model_name: Phi-4-Argunaut-1-SPIN-dev1
 tags:
 - generated_from_trainer
-- trl
 - dpo
+- trl
 licence: license
 ---
 
@@ -26,7 +26,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/ggbetz/argunauts-training/runs/rw9hb1r5) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/ggbetz/argunauts-training/runs/xxqqp8c6) 
 
 
 This model was trained with DPO, a method introduced in [Direct Preference Optimization: Your Language Model is Secretly a Reward Model](https://huggingface.co/papers/2305.18290).

all_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 2.0,
     "total_flos": 0.0,
-    "train_loss": 0.4061378458120527,
-    "train_runtime": 4146.7376,
-    "train_samples": 4358,
-    "train_samples_per_second": 2.102,
-    "train_steps_per_second": 0.066
+    "train_loss": 0.2737119815832267,
+    "train_runtime": 3862.0227,
+    "train_samples": 5211,
+    "train_samples_per_second": 2.699,
+    "train_steps_per_second": 0.084
 }

model-00001-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0790253cde713484954c768d771b249fa96bdec72fcb90a7fe61308afde03ea8
+oid sha256:26e24236fbb6eb1b1dd08f45b733e7d11a8f5c30ec2e27ba0fe29630daa7329f
 size 4933658528

model-00002-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa0e110663bff5a9b4733400e4adf94cf1b21a373b4a8963e5ecfb8e93df2337
+oid sha256:e4c2e57ad5cf1f010ba9d6c84797f82e490bdd911e169e71727635a9cc7bda35
 size 4954693112

model-00003-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4d64d5643daed3ba9e7ad668ff72d194da23b86cf933075267ead23542ea23e8
+oid sha256:e3be77aeecea5c0f1fb1f44d9d6f09f2018780173dbcafce90010ab7b77c2b53
 size 4902243992

model-00004-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a9fb96293773e40d31b31b72ac3616598c09c6d178ab2f072a1311c0035fc0ad
+oid sha256:ebbf621f807796710922fbd0cb750e413d93ee8185e892352dfb0c1982016a49
 size 4954672440

model-00005-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:46c2df5a5a7713774603a6df2017de6b1488ae1cad2136ceb12a71944ffe551f
+oid sha256:18945e8741d0e6a9a5b24addd883d59e87383c2f1f09bccae6ee72f7f3e6fcee
 size 4954672432

model-00006-of-00006.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:dba5dbed24be276d6511cdc0832ec499503a9d77a073fcf70ed183adf92bdabf
+oid sha256:e0c5346da8c3f1abbfdfe35170c7457618b4f2208b3f5cb42df08f012551c15b
 size 4619116224

train_results.json

@@ -1,9 +1,9 @@
 {
     "epoch": 2.0,
     "total_flos": 0.0,
-    "train_loss": 0.4061378458120527,
-    "train_runtime": 4146.7376,
-    "train_samples": 4358,
-    "train_samples_per_second": 2.102,
-    "train_steps_per_second": 0.066
+    "train_loss": 0.2737119815832267,
+    "train_runtime": 3862.0227,
+    "train_samples": 5211,
+    "train_samples_per_second": 2.699,
+    "train_steps_per_second": 0.084
 }

trainer_state.json

@@ -4,833 +4,998 @@
   "best_model_checkpoint": null,
   "epoch": 2.0,
   "eval_steps": 500,
-  "global_step": 274,
+  "global_step": 326,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.03669724770642202,
-      "grad_norm": 33.22876437768799,
-      "learning_rate": 1.4285714285714285e-07,
-      "logits/chosen": -1.9453125,
-      "logits/rejected": -1.8650391101837158,
-      "logps/chosen": -187.99374389648438,
-      "logps/rejected": -300.4921875,
-      "loss": 0.5688,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 0.761932373046875,
-      "rewards/margins": 1.7284057140350342,
-      "rewards/rejected": -0.96636962890625,
+      "epoch": 0.03069838833461243,
+      "grad_norm": 11.121566696073334,
+      "learning_rate": 1.176470588235294e-07,
+      "logits/chosen": -2.22265625,
+      "logits/rejected": -2.1064453125,
+      "logps/chosen": -227.35311889648438,
+      "logps/rejected": -388.5625,
+      "loss": 0.3583,
+      "rewards/accuracies": 0.793749988079071,
+      "rewards/chosen": 1.9202148914337158,
+      "rewards/margins": 3.3916258811950684,
+      "rewards/rejected": -1.4735596179962158,
       "step": 5
     },
     {
-      "epoch": 0.07339449541284404,
-      "grad_norm": 19.253905224243972,
-      "learning_rate": 3.2142857142857145e-07,
-      "logits/chosen": -1.878515601158142,
-      "logits/rejected": -1.787695288658142,
-      "logps/chosen": -177.8874969482422,
-      "logps/rejected": -218.62344360351562,
-      "loss": 0.5688,
-      "rewards/accuracies": 0.7562500238418579,
-      "rewards/chosen": 0.761035144329071,
-      "rewards/margins": 1.498071312904358,
-      "rewards/rejected": -0.7366088628768921,
+      "epoch": 0.06139677666922486,
+      "grad_norm": 20.63353450702747,
+      "learning_rate": 2.6470588235294114e-07,
+      "logits/chosen": -2.253124952316284,
+      "logits/rejected": -2.0814452171325684,
+      "logps/chosen": -248.7624969482422,
+      "logps/rejected": -484.30938720703125,
+      "loss": 0.3918,
+      "rewards/accuracies": 0.824999988079071,
+      "rewards/chosen": 2.1537842750549316,
+      "rewards/margins": 4.004614353179932,
+      "rewards/rejected": -1.8481566905975342,
       "step": 10
     },
     {
-      "epoch": 0.11009174311926606,
-      "grad_norm": 20.635311940565074,
-      "learning_rate": 5e-07,
-      "logits/chosen": -1.910546898841858,
-      "logits/rejected": -1.766992211341858,
-      "logps/chosen": -245.59375,
-      "logps/rejected": -357.078125,
-      "loss": 0.4196,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.6919189691543579,
-      "rewards/margins": 1.993749976158142,
-      "rewards/rejected": -1.2986023426055908,
+      "epoch": 0.0920951650038373,
+      "grad_norm": 16.123416834236878,
+      "learning_rate": 4.117647058823529e-07,
+      "logits/chosen": -2.2552733421325684,
+      "logits/rejected": -2.0552735328674316,
+      "logps/chosen": -299.7796936035156,
+      "logps/rejected": -378.203125,
+      "loss": 0.3305,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": 2.6285157203674316,
+      "rewards/margins": 3.8902344703674316,
+      "rewards/rejected": -1.26220703125,
       "step": 15
     },
     {
-      "epoch": 0.14678899082568808,
-      "grad_norm": 22.286628210174552,
-      "learning_rate": 4.903846153846153e-07,
-      "logits/chosen": -1.884765625,
-      "logits/rejected": -1.784570336341858,
-      "logps/chosen": -230.64218139648438,
-      "logps/rejected": -226.67343139648438,
-      "loss": 0.4481,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.9842529296875,
-      "rewards/margins": 1.7820312976837158,
-      "rewards/rejected": -0.797027587890625,
+      "epoch": 0.12279355333844973,
+      "grad_norm": 12.29166671815314,
+      "learning_rate": 4.967637540453074e-07,
+      "logits/chosen": -2.2583985328674316,
+      "logits/rejected": -2.017773389816284,
+      "logps/chosen": -316.33123779296875,
+      "logps/rejected": -459.39373779296875,
+      "loss": 0.294,
+      "rewards/accuracies": 0.8374999761581421,
+      "rewards/chosen": 2.122448682785034,
+      "rewards/margins": 4.082617282867432,
+      "rewards/rejected": -1.956140160560608,
       "step": 20
     },
     {
-      "epoch": 0.1834862385321101,
-      "grad_norm": 15.503780898074503,
-      "learning_rate": 4.807692307692307e-07,
-      "logits/chosen": -1.941796898841858,
-      "logits/rejected": -1.8427734375,
-      "logps/chosen": -174.4609375,
-      "logps/rejected": -234.84530639648438,
-      "loss": 0.4752,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 0.92529296875,
-      "rewards/margins": 1.6923644542694092,
-      "rewards/rejected": -0.76837158203125,
+      "epoch": 0.15349194167306215,
+      "grad_norm": 17.064517327038068,
+      "learning_rate": 4.886731391585761e-07,
+      "logits/chosen": -2.290234327316284,
+      "logits/rejected": -1.9728515148162842,
+      "logps/chosen": -361.1000061035156,
+      "logps/rejected": -554.6124877929688,
+      "loss": 0.2684,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 2.3677735328674316,
+      "rewards/margins": 4.116113185882568,
+      "rewards/rejected": -1.744531273841858,
       "step": 25
     },
     {
-      "epoch": 0.22018348623853212,
-      "grad_norm": 24.666179033188698,
-      "learning_rate": 4.711538461538461e-07,
-      "logits/chosen": -1.963281273841858,
-      "logits/rejected": -1.7822265625,
-      "logps/chosen": -209.16250610351562,
-      "logps/rejected": -469.15625,
-      "loss": 0.4731,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.8801330327987671,
-      "rewards/margins": 1.8875243663787842,
-      "rewards/rejected": -1.007788062095642,
+      "epoch": 0.1841903300076746,
+      "grad_norm": 11.397428856197525,
+      "learning_rate": 4.805825242718447e-07,
+      "logits/chosen": -2.171679735183716,
+      "logits/rejected": -1.9724609851837158,
+      "logps/chosen": -302.3374938964844,
+      "logps/rejected": -366.83123779296875,
+      "loss": 0.3434,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 1.95025634765625,
+      "rewards/margins": 3.369799852371216,
+      "rewards/rejected": -1.4193115234375,
       "step": 30
     },
     {
-      "epoch": 0.25688073394495414,
-      "grad_norm": 39.31686614101177,
-      "learning_rate": 4.6153846153846156e-07,
-      "logits/chosen": -1.9373047351837158,
-      "logits/rejected": -1.843164086341858,
-      "logps/chosen": -229.640625,
-      "logps/rejected": -256.80157470703125,
-      "loss": 0.4954,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.9551330804824829,
-      "rewards/margins": 1.919677734375,
-      "rewards/rejected": -0.9655059576034546,
+      "epoch": 0.21488871834228704,
+      "grad_norm": 17.965759727474417,
+      "learning_rate": 4.724919093851132e-07,
+      "logits/chosen": -2.247851610183716,
+      "logits/rejected": -2.120898485183716,
+      "logps/chosen": -242.43594360351562,
+      "logps/rejected": -474.0562438964844,
+      "loss": 0.291,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 2.177441358566284,
+      "rewards/margins": 3.690624952316284,
+      "rewards/rejected": -1.5150024890899658,
       "step": 35
     },
     {
-      "epoch": 0.29357798165137616,
-      "grad_norm": 25.350998609878594,
-      "learning_rate": 4.519230769230769e-07,
-      "logits/chosen": -1.98046875,
-      "logits/rejected": -1.8894531726837158,
-      "logps/chosen": -206.1374969482422,
-      "logps/rejected": -301.48126220703125,
-      "loss": 0.4124,
-      "rewards/accuracies": 0.7875000238418579,
-      "rewards/chosen": 1.19140625,
-      "rewards/margins": 2.3330321311950684,
-      "rewards/rejected": -1.14300537109375,
+      "epoch": 0.24558710667689945,
+      "grad_norm": 16.842023758004515,
+      "learning_rate": 4.6440129449838184e-07,
+      "logits/chosen": -2.28515625,
+      "logits/rejected": -2.058789014816284,
+      "logps/chosen": -255.53125,
+      "logps/rejected": -484.47186279296875,
+      "loss": 0.3384,
+      "rewards/accuracies": 0.800000011920929,
+      "rewards/chosen": 2.153881788253784,
+      "rewards/margins": 3.767773389816284,
+      "rewards/rejected": -1.615258812904358,
       "step": 40
     },
     {
-      "epoch": 0.3302752293577982,
-      "grad_norm": 11.623966835415967,
-      "learning_rate": 4.423076923076923e-07,
-      "logits/chosen": -1.8582031726837158,
-      "logits/rejected": -1.7023437023162842,
-      "logps/chosen": -171.0906219482422,
-      "logps/rejected": -261.80938720703125,
-      "loss": 0.4269,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 1.000451683998108,
-      "rewards/margins": 1.933935523033142,
-      "rewards/rejected": -0.9315429925918579,
+      "epoch": 0.2762854950115119,
+      "grad_norm": 22.9317252710755,
+      "learning_rate": 4.563106796116505e-07,
+      "logits/chosen": -2.24609375,
+      "logits/rejected": -2.001171827316284,
+      "logps/chosen": -327.17498779296875,
+      "logps/rejected": -405.9375,
+      "loss": 0.2816,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 2.350268602371216,
+      "rewards/margins": 4.024218559265137,
+      "rewards/rejected": -1.6744873523712158,
       "step": 45
     },
     {
-      "epoch": 0.3669724770642202,
-      "grad_norm": 7.730087812810576,
-      "learning_rate": 4.326923076923077e-07,
-      "logits/chosen": -1.804296851158142,
-      "logits/rejected": -1.739648461341858,
-      "logps/chosen": -143.0078125,
-      "logps/rejected": -210.30313110351562,
-      "loss": 0.3994,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": 1.30889892578125,
-      "rewards/margins": 1.947778344154358,
-      "rewards/rejected": -0.6377807855606079,
+      "epoch": 0.3069838833461243,
+      "grad_norm": 15.527675742487922,
+      "learning_rate": 4.4822006472491906e-07,
+      "logits/chosen": -2.308789014816284,
+      "logits/rejected": -2.0882811546325684,
+      "logps/chosen": -244.3156280517578,
+      "logps/rejected": -342.50311279296875,
+      "loss": 0.3627,
+      "rewards/accuracies": 0.8187500238418579,
+      "rewards/chosen": 2.4039337635040283,
+      "rewards/margins": 3.5143370628356934,
+      "rewards/rejected": -1.1082031726837158,
       "step": 50
     },
     {
-      "epoch": 0.4036697247706422,
-      "grad_norm": 25.104254895206235,
-      "learning_rate": 4.2307692307692304e-07,
-      "logits/chosen": -1.8533203601837158,
-      "logits/rejected": -1.7498047351837158,
-      "logps/chosen": -167.6453094482422,
-      "logps/rejected": -299.8343811035156,
-      "loss": 0.5257,
-      "rewards/accuracies": 0.762499988079071,
-      "rewards/chosen": 1.0325927734375,
-      "rewards/margins": 1.3797729015350342,
-      "rewards/rejected": -0.34746092557907104,
+      "epoch": 0.3376822716807368,
+      "grad_norm": 21.127864486539234,
+      "learning_rate": 4.4012944983818767e-07,
+      "logits/chosen": -2.2890625,
+      "logits/rejected": -2.038281202316284,
+      "logps/chosen": -308.95001220703125,
+      "logps/rejected": -414.5375061035156,
+      "loss": 0.2959,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 2.238818407058716,
+      "rewards/margins": 3.924755811691284,
+      "rewards/rejected": -1.6855957508087158,
       "step": 55
     },
     {
-      "epoch": 0.44036697247706424,
-      "grad_norm": 46.19801478111461,
-      "learning_rate": 4.134615384615384e-07,
-      "logits/chosen": -1.813085913658142,
-      "logits/rejected": -1.788671851158142,
-      "logps/chosen": -266.36407470703125,
-      "logps/rejected": -231.34530639648438,
-      "loss": 0.553,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 1.023596167564392,
-      "rewards/margins": 1.6953766345977783,
-      "rewards/rejected": -0.6730102300643921,
+      "epoch": 0.3683806600153492,
+      "grad_norm": 52.75258924580777,
+      "learning_rate": 4.320388349514563e-07,
+      "logits/chosen": -2.2689452171325684,
+      "logits/rejected": -2.0503907203674316,
+      "logps/chosen": -264.5375061035156,
+      "logps/rejected": -451.8687438964844,
+      "loss": 0.3218,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 1.5753967761993408,
+      "rewards/margins": 4.02734375,
+      "rewards/rejected": -2.451855421066284,
       "step": 60
     },
     {
-      "epoch": 0.47706422018348627,
-      "grad_norm": 28.971320495320786,
-      "learning_rate": 4.0384615384615386e-07,
-      "logits/chosen": -1.806640625,
-      "logits/rejected": -1.747656226158142,
-      "logps/chosen": -214.56875610351562,
-      "logps/rejected": -283.62969970703125,
-      "loss": 0.4405,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.7670348882675171,
-      "rewards/margins": 1.659521460533142,
-      "rewards/rejected": -0.891857922077179,
+      "epoch": 0.3990790483499616,
+      "grad_norm": 20.196062332868237,
+      "learning_rate": 4.239482200647249e-07,
+      "logits/chosen": -2.2076172828674316,
+      "logits/rejected": -1.9767577648162842,
+      "logps/chosen": -292.6937561035156,
+      "logps/rejected": -459.34375,
+      "loss": 0.3184,
+      "rewards/accuracies": 0.875,
+      "rewards/chosen": 2.086029052734375,
+      "rewards/margins": 3.979687452316284,
+      "rewards/rejected": -1.89422607421875,
       "step": 65
     },
     {
-      "epoch": 0.5137614678899083,
-      "grad_norm": 16.80170550474919,
-      "learning_rate": 3.942307692307692e-07,
-      "logits/chosen": -1.874609351158142,
-      "logits/rejected": -1.8097655773162842,
-      "logps/chosen": -192.9718780517578,
-      "logps/rejected": -241.9343719482422,
-      "loss": 0.4856,
-      "rewards/accuracies": 0.75,
-      "rewards/chosen": 1.166839599609375,
-      "rewards/margins": 1.6161620616912842,
-      "rewards/rejected": -0.45008546113967896,
+      "epoch": 0.4297774366845741,
+      "grad_norm": 20.093203861551977,
+      "learning_rate": 4.158576051779935e-07,
+      "logits/chosen": -2.251757860183716,
+      "logits/rejected": -2.056640625,
+      "logps/chosen": -290.328125,
+      "logps/rejected": -389.9156188964844,
+      "loss": 0.2854,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 2.213456630706787,
+      "rewards/margins": 3.6863036155700684,
+      "rewards/rejected": -1.474511742591858,
       "step": 70
     },
     {
-      "epoch": 0.5504587155963303,
-      "grad_norm": 26.406721071076017,
-      "learning_rate": 3.8461538461538463e-07,
-      "logits/chosen": -1.87109375,
-      "logits/rejected": -1.772851586341858,
-      "logps/chosen": -171.234375,
-      "logps/rejected": -202.3640594482422,
-      "loss": 0.4186,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 1.05938720703125,
-      "rewards/margins": 1.654272437095642,
-      "rewards/rejected": -0.5958007574081421,
+      "epoch": 0.4604758250191865,
+      "grad_norm": 19.79634574619263,
+      "learning_rate": 4.077669902912621e-07,
+      "logits/chosen": -2.2406249046325684,
+      "logits/rejected": -2.00390625,
+      "logps/chosen": -293.62811279296875,
+      "logps/rejected": -435.84375,
+      "loss": 0.2902,
+      "rewards/accuracies": 0.84375,
+      "rewards/chosen": 1.892968773841858,
+      "rewards/margins": 3.774218797683716,
+      "rewards/rejected": -1.883551001548767,
       "step": 75
     },
     {
-      "epoch": 0.5871559633027523,
-      "grad_norm": 12.695931204244943,
-      "learning_rate": 3.75e-07,
-      "logits/chosen": -1.9052734375,
-      "logits/rejected": -1.784570336341858,
-      "logps/chosen": -179.98281860351562,
-      "logps/rejected": -362.39373779296875,
-      "loss": 0.4228,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.9754394292831421,
-      "rewards/margins": 1.9592773914337158,
-      "rewards/rejected": -0.9834350347518921,
+      "epoch": 0.4911742133537989,
+      "grad_norm": 19.181029218450373,
+      "learning_rate": 3.9967637540453073e-07,
+      "logits/chosen": -2.2816405296325684,
+      "logits/rejected": -2.0777344703674316,
+      "logps/chosen": -301.49688720703125,
+      "logps/rejected": -399.69061279296875,
+      "loss": 0.2902,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 2.0974364280700684,
+      "rewards/margins": 3.5245604515075684,
+      "rewards/rejected": -1.4281127452850342,
       "step": 80
     },
     {
-      "epoch": 0.6238532110091743,
-      "grad_norm": 11.877707612125597,
-      "learning_rate": 3.6538461538461534e-07,
-      "logits/chosen": -1.870703101158142,
-      "logits/rejected": -1.8044922351837158,
-      "logps/chosen": -184.58438110351562,
-      "logps/rejected": -252.95938110351562,
-      "loss": 0.4589,
-      "rewards/accuracies": 0.768750011920929,
-      "rewards/chosen": 0.9694183468818665,
-      "rewards/margins": 1.938848853111267,
-      "rewards/rejected": -0.9698241949081421,
+      "epoch": 0.5218726016884113,
+      "grad_norm": 14.675252239263,
+      "learning_rate": 3.9158576051779934e-07,
+      "logits/chosen": -2.2740235328674316,
+      "logits/rejected": -2.0689454078674316,
+      "logps/chosen": -294.4234313964844,
+      "logps/rejected": -564.1343994140625,
+      "loss": 0.2916,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 1.751062035560608,
+      "rewards/margins": 3.894702196121216,
+      "rewards/rejected": -2.143725633621216,
       "step": 85
     },
     {
-      "epoch": 0.6605504587155964,
-      "grad_norm": 24.425154245858224,
-      "learning_rate": 3.557692307692308e-07,
-      "logits/chosen": -1.81640625,
-      "logits/rejected": -1.730859398841858,
-      "logps/chosen": -131.6875,
-      "logps/rejected": -210.6374969482422,
-      "loss": 0.3979,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 1.023162841796875,
-      "rewards/margins": 1.620874047279358,
-      "rewards/rejected": -0.597125232219696,
+      "epoch": 0.5525709900230238,
+      "grad_norm": 23.299079990362248,
+      "learning_rate": 3.8349514563106795e-07,
+      "logits/chosen": -2.2728514671325684,
+      "logits/rejected": -2.0404295921325684,
+      "logps/chosen": -317.50311279296875,
+      "logps/rejected": -462.9156188964844,
+      "loss": 0.2674,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 2.2327880859375,
+      "rewards/margins": 3.934765577316284,
+      "rewards/rejected": -1.7007324695587158,
       "step": 90
     },
     {
-      "epoch": 0.6972477064220184,
-      "grad_norm": 11.31997736864757,
-      "learning_rate": 3.461538461538461e-07,
-      "logits/chosen": -1.8603515625,
-      "logits/rejected": -1.7619140148162842,
-      "logps/chosen": -187.9718780517578,
-      "logps/rejected": -293.79998779296875,
-      "loss": 0.4306,
-      "rewards/accuracies": 0.793749988079071,
-      "rewards/chosen": 0.9427169561386108,
-      "rewards/margins": 2.0055174827575684,
-      "rewards/rejected": -1.0625121593475342,
+      "epoch": 0.5832693783576363,
+      "grad_norm": 55.83907641753374,
+      "learning_rate": 3.754045307443365e-07,
+      "logits/chosen": -2.2884764671325684,
+      "logits/rejected": -2.100390672683716,
+      "logps/chosen": -415.45623779296875,
+      "logps/rejected": -376.41876220703125,
+      "loss": 0.3104,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 2.108752489089966,
+      "rewards/margins": 4.00244140625,
+      "rewards/rejected": -1.894140601158142,
       "step": 95
     },
     {
-      "epoch": 0.7339449541284404,
-      "grad_norm": 16.984533533471815,
-      "learning_rate": 3.3653846153846154e-07,
-      "logits/chosen": -1.8857421875,
-      "logits/rejected": -1.783789038658142,
-      "logps/chosen": -161.58749389648438,
-      "logps/rejected": -282.7953186035156,
-      "loss": 0.4301,
-      "rewards/accuracies": 0.78125,
-      "rewards/chosen": 0.9314972162246704,
-      "rewards/margins": 1.686669945716858,
-      "rewards/rejected": -0.7535156011581421,
+      "epoch": 0.6139677666922486,
+      "grad_norm": 30.671605767336874,
+      "learning_rate": 3.673139158576052e-07,
+      "logits/chosen": -2.287890672683716,
+      "logits/rejected": -2.02734375,
+      "logps/chosen": -316.25,
+      "logps/rejected": -460.70001220703125,
+      "loss": 0.2841,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 1.875756859779358,
+      "rewards/margins": 3.8394532203674316,
+      "rewards/rejected": -1.9620239734649658,
       "step": 100
     },
     {
-      "epoch": 0.7706422018348624,
-      "grad_norm": 22.070587300610953,
-      "learning_rate": 3.269230769230769e-07,
-      "logits/chosen": -1.9326171875,
-      "logits/rejected": -1.821679711341858,
-      "logps/chosen": -218.49374389648438,
-      "logps/rejected": -321.09375,
-      "loss": 0.3785,
+      "epoch": 0.6446661550268611,
+      "grad_norm": 20.82501262179368,
+      "learning_rate": 3.592233009708738e-07,
+      "logits/chosen": -2.256054639816284,
+      "logits/rejected": -2.04296875,
+      "logps/chosen": -324.45623779296875,
+      "logps/rejected": -504.0406188964844,
+      "loss": 0.2799,
       "rewards/accuracies": 0.856249988079071,
-      "rewards/chosen": 1.01708984375,
-      "rewards/margins": 2.0724120140075684,
-      "rewards/rejected": -1.054956078529358,
+      "rewards/chosen": 2.1114563941955566,
+      "rewards/margins": 4.019073486328125,
+      "rewards/rejected": -1.9062011241912842,
       "step": 105
     },
     {
-      "epoch": 0.8073394495412844,
-      "grad_norm": 15.404403703735678,
-      "learning_rate": 3.1730769230769225e-07,
-      "logits/chosen": -1.875,
-      "logits/rejected": -1.7683594226837158,
-      "logps/chosen": -223.3468780517578,
-      "logps/rejected": -269.09063720703125,
-      "loss": 0.3714,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": 0.7868896722793579,
-      "rewards/margins": 1.91033935546875,
-      "rewards/rejected": -1.123510718345642,
+      "epoch": 0.6753645433614736,
+      "grad_norm": 33.80916604419537,
+      "learning_rate": 3.5113268608414234e-07,
+      "logits/chosen": -2.2220702171325684,
+      "logits/rejected": -2.0341796875,
+      "logps/chosen": -315.27032470703125,
+      "logps/rejected": -288.078125,
+      "loss": 0.2677,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 2.1306395530700684,
+      "rewards/margins": 3.439257860183716,
+      "rewards/rejected": -1.3082396984100342,
       "step": 110
     },
     {
-      "epoch": 0.8440366972477065,
-      "grad_norm": 33.94178443490644,
-      "learning_rate": 3.076923076923077e-07,
-      "logits/chosen": -1.8820312023162842,
-      "logits/rejected": -1.7746093273162842,
-      "logps/chosen": -200.58438110351562,
-      "logps/rejected": -195.2468719482422,
-      "loss": 0.4604,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 0.874560534954071,
-      "rewards/margins": 1.737646460533142,
-      "rewards/rejected": -0.8634979128837585,
+      "epoch": 0.7060629316960859,
+      "grad_norm": 22.479271981046605,
+      "learning_rate": 3.4304207119741096e-07,
+      "logits/chosen": -2.2115235328674316,
+      "logits/rejected": -1.968359351158142,
+      "logps/chosen": -346.90625,
+      "logps/rejected": -690.7062377929688,
+      "loss": 0.3067,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 1.858984351158142,
+      "rewards/margins": 4.519238471984863,
+      "rewards/rejected": -2.662036180496216,
       "step": 115
     },
     {
-      "epoch": 0.8807339449541285,
-      "grad_norm": 13.077979257118965,
-      "learning_rate": 2.980769230769231e-07,
-      "logits/chosen": -1.856054663658142,
-      "logits/rejected": -1.794531226158142,
-      "logps/chosen": -151.5437469482422,
-      "logps/rejected": -196.953125,
-      "loss": 0.3459,
-      "rewards/accuracies": 0.862500011920929,
-      "rewards/chosen": 1.229943871498108,
-      "rewards/margins": 2.440661668777466,
-      "rewards/rejected": -1.2105392217636108,
+      "epoch": 0.7367613200306984,
+      "grad_norm": 22.82747748987554,
+      "learning_rate": 3.349514563106796e-07,
+      "logits/chosen": -2.2685546875,
+      "logits/rejected": -2.046093702316284,
+      "logps/chosen": -322.59375,
+      "logps/rejected": -554.1500244140625,
+      "loss": 0.3413,
+      "rewards/accuracies": 0.8687499761581421,
+      "rewards/chosen": 1.87310791015625,
+      "rewards/margins": 4.309618949890137,
+      "rewards/rejected": -2.433239698410034,
       "step": 120
     },
     {
-      "epoch": 0.9174311926605505,
-      "grad_norm": 30.51657692857472,
-      "learning_rate": 2.884615384615384e-07,
-      "logits/chosen": -1.847265601158142,
-      "logits/rejected": -1.7861328125,
-      "logps/chosen": -173.4406280517578,
-      "logps/rejected": -255.5749969482422,
-      "loss": 0.4275,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.864672839641571,
-      "rewards/margins": 1.76495361328125,
-      "rewards/rejected": -0.89959716796875,
+      "epoch": 0.7674597083653109,
+      "grad_norm": 22.397741857744723,
+      "learning_rate": 3.2686084142394823e-07,
+      "logits/chosen": -2.2513670921325684,
+      "logits/rejected": -2.0927734375,
+      "logps/chosen": -302.29998779296875,
+      "logps/rejected": -564.328125,
+      "loss": 0.2886,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 2.023284912109375,
+      "rewards/margins": 3.8551268577575684,
+      "rewards/rejected": -1.830969214439392,
       "step": 125
     },
     {
-      "epoch": 0.9541284403669725,
-      "grad_norm": 13.780160219092938,
-      "learning_rate": 2.7884615384615384e-07,
-      "logits/chosen": -1.877343773841858,
-      "logits/rejected": -1.738671898841858,
-      "logps/chosen": -214.515625,
-      "logps/rejected": -354.0171813964844,
-      "loss": 0.3844,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.858630359172821,
-      "rewards/margins": 2.1751465797424316,
-      "rewards/rejected": -1.316162109375,
+      "epoch": 0.7981580966999232,
+      "grad_norm": 8.578228102008167,
+      "learning_rate": 3.187702265372168e-07,
+      "logits/chosen": -2.2466797828674316,
+      "logits/rejected": -2.034374952316284,
+      "logps/chosen": -324.86248779296875,
+      "logps/rejected": -450.6187438964844,
+      "loss": 0.2982,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 2.017773389816284,
+      "rewards/margins": 4.181738376617432,
+      "rewards/rejected": -2.1627564430236816,
       "step": 130
     },
     {
-      "epoch": 0.9908256880733946,
-      "grad_norm": 27.1013043867592,
-      "learning_rate": 2.692307692307692e-07,
-      "logits/chosen": -1.806054711341858,
-      "logits/rejected": -1.7755858898162842,
-      "logps/chosen": -256.57501220703125,
-      "logps/rejected": -198.11874389648438,
-      "loss": 0.4336,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": 0.836621105670929,
-      "rewards/margins": 1.724884033203125,
-      "rewards/rejected": -0.8876087069511414,
+      "epoch": 0.8288564850345357,
+      "grad_norm": 9.589882721872858,
+      "learning_rate": 3.106796116504854e-07,
+      "logits/chosen": -2.204882860183716,
+      "logits/rejected": -2.0736327171325684,
+      "logps/chosen": -283.6703186035156,
+      "logps/rejected": -438.23126220703125,
+      "loss": 0.3102,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 1.94146728515625,
+      "rewards/margins": 3.675122022628784,
+      "rewards/rejected": -1.735009789466858,
       "step": 135
     },
     {
-      "epoch": 1.0220183486238532,
-      "grad_norm": 13.031886237591982,
-      "learning_rate": 2.596153846153846e-07,
-      "logits/chosen": -1.8766084909439087,
-      "logits/rejected": -1.7973345518112183,
-      "logps/chosen": -228.58456420898438,
-      "logps/rejected": -230.00735473632812,
-      "loss": 0.4131,
-      "rewards/accuracies": 0.8602941036224365,
-      "rewards/chosen": 0.6476907134056091,
-      "rewards/margins": 1.6508358716964722,
-      "rewards/rejected": -1.0023910999298096,
+      "epoch": 0.8595548733691482,
+      "grad_norm": 26.716783170877356,
+      "learning_rate": 3.0258899676375407e-07,
+      "logits/chosen": -2.226757764816284,
+      "logits/rejected": -2.0386719703674316,
+      "logps/chosen": -330.4624938964844,
+      "logps/rejected": -520.3562622070312,
+      "loss": 0.2945,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 1.6619141101837158,
+      "rewards/margins": 3.725292921066284,
+      "rewards/rejected": -2.062939405441284,
       "step": 140
     },
     {
-      "epoch": 1.0587155963302752,
-      "grad_norm": 7.580044936631597,
-      "learning_rate": 2.5e-07,
-      "logits/chosen": -1.8037109375,
-      "logits/rejected": -1.7208983898162842,
-      "logps/chosen": -170.09375,
-      "logps/rejected": -344.7437438964844,
-      "loss": 0.4014,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": 1.0894775390625,
-      "rewards/margins": 1.9444580078125,
-      "rewards/rejected": -0.85498046875,
+      "epoch": 0.8902532617037605,
+      "grad_norm": 28.642400942966432,
+      "learning_rate": 2.944983818770226e-07,
+      "logits/chosen": -2.2412109375,
+      "logits/rejected": -2.065624952316284,
+      "logps/chosen": -288.29998779296875,
+      "logps/rejected": -337.26873779296875,
+      "loss": 0.3528,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 1.7783691883087158,
+      "rewards/margins": 3.4496827125549316,
+      "rewards/rejected": -1.66796875,
       "step": 145
     },
     {
-      "epoch": 1.0954128440366973,
-      "grad_norm": 15.690513831070637,
-      "learning_rate": 2.4038461538461537e-07,
-      "logits/chosen": -1.822656273841858,
-      "logits/rejected": -1.779296875,
-      "logps/chosen": -248.88125610351562,
-      "logps/rejected": -325.3500061035156,
-      "loss": 0.3528,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.877734363079071,
-      "rewards/margins": 2.3188233375549316,
-      "rewards/rejected": -1.4415161609649658,
+      "epoch": 0.920951650038373,
+      "grad_norm": 20.19215767876997,
+      "learning_rate": 2.8640776699029124e-07,
+      "logits/chosen": -2.296093702316284,
+      "logits/rejected": -2.0794920921325684,
+      "logps/chosen": -275.4468688964844,
+      "logps/rejected": -425.65313720703125,
+      "loss": 0.2387,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 2.306689500808716,
+      "rewards/margins": 4.460400581359863,
+      "rewards/rejected": -2.156982421875,
       "step": 150
     },
     {
-      "epoch": 1.1321100917431193,
-      "grad_norm": 15.215220163021907,
-      "learning_rate": 2.3076923076923078e-07,
-      "logits/chosen": -1.85546875,
-      "logits/rejected": -1.762109398841858,
-      "logps/chosen": -178.05313110351562,
-      "logps/rejected": -250.49374389648438,
-      "loss": 0.4006,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.9465576410293579,
-      "rewards/margins": 1.915624976158142,
-      "rewards/rejected": -0.970104992389679,
+      "epoch": 0.9516500383729855,
+      "grad_norm": 18.41603462189631,
+      "learning_rate": 2.783171521035599e-07,
+      "logits/chosen": -2.2021484375,
+      "logits/rejected": -2.01953125,
+      "logps/chosen": -431.5687561035156,
+      "logps/rejected": -383.07501220703125,
+      "loss": 0.3324,
+      "rewards/accuracies": 0.887499988079071,
+      "rewards/chosen": 2.0430665016174316,
+      "rewards/margins": 4.26806640625,
+      "rewards/rejected": -2.2254638671875,
       "step": 155
     },
     {
-      "epoch": 1.1688073394495413,
-      "grad_norm": 18.54494454979772,
-      "learning_rate": 2.2115384615384614e-07,
-      "logits/chosen": -1.858007788658142,
-      "logits/rejected": -1.782812476158142,
-      "logps/chosen": -179.8249969482422,
-      "logps/rejected": -342.28125,
-      "loss": 0.378,
-      "rewards/accuracies": 0.8062499761581421,
-      "rewards/chosen": 0.7204223871231079,
-      "rewards/margins": 2.033404588699341,
-      "rewards/rejected": -1.3119995594024658,
+      "epoch": 0.9823484267075978,
+      "grad_norm": 22.4727442260727,
+      "learning_rate": 2.7022653721682846e-07,
+      "logits/chosen": -2.190234422683716,
+      "logits/rejected": -2.022265672683716,
+      "logps/chosen": -301.4593811035156,
+      "logps/rejected": -401.4593811035156,
+      "loss": 0.2658,
+      "rewards/accuracies": 0.856249988079071,
+      "rewards/chosen": 2.146771192550659,
+      "rewards/margins": 4.336230278015137,
+      "rewards/rejected": -2.1904234886169434,
       "step": 160
     },
     {
-      "epoch": 1.2055045871559633,
-      "grad_norm": 11.492838106006257,
-      "learning_rate": 2.1153846153846152e-07,
-      "logits/chosen": -1.938867211341858,
-      "logits/rejected": -1.78515625,
-      "logps/chosen": -210.4499969482422,
-      "logps/rejected": -346.09375,
-      "loss": 0.3104,
-      "rewards/accuracies": 0.8687499761581421,
-      "rewards/chosen": 0.9412597417831421,
-      "rewards/margins": 2.375317335128784,
-      "rewards/rejected": -1.43389892578125,
+      "epoch": 1.012279355333845,
+      "grad_norm": 29.045746191502413,
+      "learning_rate": 2.6213592233009707e-07,
+      "logits/chosen": -2.2520031929016113,
+      "logits/rejected": -1.9837740659713745,
+      "logps/chosen": -268.5384521484375,
+      "logps/rejected": -455.69232177734375,
+      "loss": 0.2525,
+      "rewards/accuracies": 0.9081196188926697,
+      "rewards/chosen": 2.257117748260498,
+      "rewards/margins": 4.123847961425781,
+      "rewards/rejected": -1.8706743717193604,
       "step": 165
     },
     {
-      "epoch": 1.2422018348623853,
-      "grad_norm": 22.063054783212692,
-      "learning_rate": 2.0192307692307693e-07,
-      "logits/chosen": -1.927148461341858,
-      "logits/rejected": -1.8445312976837158,
-      "logps/chosen": -182.09375,
-      "logps/rejected": -195.45938110351562,
-      "loss": 0.3562,
-      "rewards/accuracies": 0.8125,
-      "rewards/chosen": 0.888519287109375,
-      "rewards/margins": 2.130175828933716,
-      "rewards/rejected": -1.2406799793243408,
+      "epoch": 1.0429777436684573,
+      "grad_norm": 41.53269802898524,
+      "learning_rate": 2.540453074433657e-07,
+      "logits/chosen": -2.285937547683716,
+      "logits/rejected": -2.0611329078674316,
+      "logps/chosen": -335.42498779296875,
+      "logps/rejected": -666.0999755859375,
+      "loss": 0.2565,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 2.12225341796875,
+      "rewards/margins": 4.858691215515137,
+      "rewards/rejected": -2.73529052734375,
       "step": 170
     },
     {
-      "epoch": 1.2788990825688074,
-      "grad_norm": 18.824995052782572,
-      "learning_rate": 1.9230769230769231e-07,
-      "logits/chosen": -1.906835913658142,
-      "logits/rejected": -1.78125,
-      "logps/chosen": -199.08438110351562,
-      "logps/rejected": -278.6812438964844,
-      "loss": 0.3636,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 0.8860107660293579,
-      "rewards/margins": 2.08349609375,
-      "rewards/rejected": -1.1968567371368408,
+      "epoch": 1.0736761320030699,
+      "grad_norm": 5.165992871948705,
+      "learning_rate": 2.459546925566343e-07,
+      "logits/chosen": -2.252148389816284,
+      "logits/rejected": -2.0345702171325684,
+      "logps/chosen": -309.3500061035156,
+      "logps/rejected": -519.1593627929688,
+      "loss": 0.2073,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 2.195666551589966,
+      "rewards/margins": 4.653515815734863,
+      "rewards/rejected": -2.455151319503784,
       "step": 175
     },
     {
-      "epoch": 1.3155963302752294,
-      "grad_norm": 5.927489409391164,
-      "learning_rate": 1.8269230769230767e-07,
-      "logits/chosen": -1.8244140148162842,
-      "logits/rejected": -1.7685546875,
-      "logps/chosen": -175.609375,
-      "logps/rejected": -202.66250610351562,
-      "loss": 0.352,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.9653075933456421,
-      "rewards/margins": 1.90960693359375,
-      "rewards/rejected": -0.943652331829071,
+      "epoch": 1.1043745203376822,
+      "grad_norm": 7.172517838598237,
+      "learning_rate": 2.378640776699029e-07,
+      "logits/chosen": -2.219921827316284,
+      "logits/rejected": -2.071093797683716,
+      "logps/chosen": -306.23126220703125,
+      "logps/rejected": -418.01251220703125,
+      "loss": 0.2665,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 2.138867139816284,
+      "rewards/margins": 4.50830078125,
+      "rewards/rejected": -2.36920166015625,
       "step": 180
     },
     {
-      "epoch": 1.3522935779816514,
-      "grad_norm": 9.770113429927585,
-      "learning_rate": 1.7307692307692305e-07,
-      "logits/chosen": -1.9005858898162842,
-      "logits/rejected": -1.786523461341858,
-      "logps/chosen": -218.546875,
-      "logps/rejected": -410.6187438964844,
-      "loss": 0.3399,
-      "rewards/accuracies": 0.824999988079071,
-      "rewards/chosen": 0.8787597417831421,
-      "rewards/margins": 2.3956542015075684,
-      "rewards/rejected": -1.516717553138733,
+      "epoch": 1.1350729086722948,
+      "grad_norm": 10.018114423357114,
+      "learning_rate": 2.297734627831715e-07,
+      "logits/chosen": -2.275195360183716,
+      "logits/rejected": -2.061328172683716,
+      "logps/chosen": -350.2749938964844,
+      "logps/rejected": -652.1687622070312,
+      "loss": 0.2009,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 1.891626000404358,
+      "rewards/margins": 4.262890815734863,
+      "rewards/rejected": -2.3704590797424316,
       "step": 185
     },
     {
-      "epoch": 1.3889908256880734,
-      "grad_norm": 17.54848745249605,
-      "learning_rate": 1.6346153846153846e-07,
-      "logits/chosen": -1.9091796875,
-      "logits/rejected": -1.811914086341858,
-      "logps/chosen": -160.11874389648438,
-      "logps/rejected": -216.6921844482422,
-      "loss": 0.3768,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.76153564453125,
-      "rewards/margins": 2.0511474609375,
-      "rewards/rejected": -1.288671851158142,
+      "epoch": 1.1657712970069072,
+      "grad_norm": 20.436381868896863,
+      "learning_rate": 2.2168284789644013e-07,
+      "logits/chosen": -2.26171875,
+      "logits/rejected": -2.015429735183716,
+      "logps/chosen": -372.1812438964844,
+      "logps/rejected": -343.4375,
+      "loss": 0.266,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 1.8646728992462158,
+      "rewards/margins": 3.8271484375,
+      "rewards/rejected": -1.962133765220642,
       "step": 190
     },
     {
-      "epoch": 1.4256880733944954,
-      "grad_norm": 10.046035163946694,
-      "learning_rate": 1.5384615384615385e-07,
-      "logits/chosen": -1.866601586341858,
-      "logits/rejected": -1.762304663658142,
-      "logps/chosen": -152.25625610351562,
-      "logps/rejected": -212.5749969482422,
-      "loss": 0.3387,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.7191314697265625,
-      "rewards/margins": 2.0878663063049316,
-      "rewards/rejected": -1.3691924810409546,
+      "epoch": 1.1964696853415195,
+      "grad_norm": 6.83979158792521,
+      "learning_rate": 2.1359223300970871e-07,
+      "logits/chosen": -2.2855467796325684,
+      "logits/rejected": -2.160351514816284,
+      "logps/chosen": -342.2406311035156,
+      "logps/rejected": -460.0093688964844,
+      "loss": 0.2465,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 1.532690405845642,
+      "rewards/margins": 3.607617139816284,
+      "rewards/rejected": -2.077587842941284,
       "step": 195
     },
     {
-      "epoch": 1.4623853211009175,
-      "grad_norm": 24.64891684255534,
-      "learning_rate": 1.442307692307692e-07,
-      "logits/chosen": -1.852148413658142,
-      "logits/rejected": -1.7707030773162842,
-      "logps/chosen": -150.7468719482422,
-      "logps/rejected": -293.0625,
-      "loss": 0.3384,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.9279540777206421,
-      "rewards/margins": 2.073071241378784,
-      "rewards/rejected": -1.145782470703125,
+      "epoch": 1.2271680736761321,
+      "grad_norm": 10.68098901348123,
+      "learning_rate": 2.0550161812297733e-07,
+      "logits/chosen": -2.2890625,
+      "logits/rejected": -2.1167969703674316,
+      "logps/chosen": -305.42657470703125,
+      "logps/rejected": -532.4812622070312,
+      "loss": 0.2422,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 1.8942139148712158,
+      "rewards/margins": 3.9365234375,
+      "rewards/rejected": -2.0408568382263184,
       "step": 200
     },
     {
-      "epoch": 1.4990825688073395,
-      "grad_norm": 10.264189832377008,
-      "learning_rate": 1.346153846153846e-07,
-      "logits/chosen": -1.8664062023162842,
-      "logits/rejected": -1.7138671875,
-      "logps/chosen": -186.28750610351562,
-      "logps/rejected": -292.2749938964844,
-      "loss": 0.3129,
-      "rewards/accuracies": 0.887499988079071,
-      "rewards/chosen": 0.9490722417831421,
-      "rewards/margins": 2.3033204078674316,
-      "rewards/rejected": -1.3525269031524658,
+      "epoch": 1.2578664620107445,
+      "grad_norm": 17.918521637264806,
+      "learning_rate": 1.9741100323624594e-07,
+      "logits/chosen": -2.105273485183716,
+      "logits/rejected": -1.9345703125,
+      "logps/chosen": -315.2796936035156,
+      "logps/rejected": -413.49688720703125,
+      "loss": 0.2416,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 2.3636536598205566,
+      "rewards/margins": 4.360058784484863,
+      "rewards/rejected": -1.9951903820037842,
       "step": 205
     },
     {
-      "epoch": 1.5357798165137615,
-      "grad_norm": 29.504068972664967,
-      "learning_rate": 1.25e-07,
-      "logits/chosen": -1.8708984851837158,
-      "logits/rejected": -1.767578125,
-      "logps/chosen": -234.1750030517578,
-      "logps/rejected": -315.8374938964844,
-      "loss": 0.3891,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 0.8542724847793579,
-      "rewards/margins": 2.066845655441284,
-      "rewards/rejected": -1.2125946283340454,
+      "epoch": 1.2885648503453568,
+      "grad_norm": 11.29949994925298,
+      "learning_rate": 1.8932038834951455e-07,
+      "logits/chosen": -2.263867139816284,
+      "logits/rejected": -2.0765624046325684,
+      "logps/chosen": -325.53436279296875,
+      "logps/rejected": -471.0687561035156,
+      "loss": 0.2416,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 2.2369384765625,
+      "rewards/margins": 4.525781154632568,
+      "rewards/rejected": -2.2864136695861816,
       "step": 210
     },
     {
-      "epoch": 1.5724770642201835,
-      "grad_norm": 11.711474643815764,
-      "learning_rate": 1.1538461538461539e-07,
-      "logits/chosen": -1.9169921875,
-      "logits/rejected": -1.769921898841858,
-      "logps/chosen": -176.625,
-      "logps/rejected": -314.76873779296875,
-      "loss": 0.3676,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 1.0230834484100342,
-      "rewards/margins": 2.096282958984375,
-      "rewards/rejected": -1.0709412097930908,
+      "epoch": 1.3192632386799694,
+      "grad_norm": 21.00338870046118,
+      "learning_rate": 1.8122977346278319e-07,
+      "logits/chosen": -2.228515625,
+      "logits/rejected": -1.9753906726837158,
+      "logps/chosen": -522.5625,
+      "logps/rejected": -537.4312744140625,
+      "loss": 0.2119,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 2.187304735183716,
+      "rewards/margins": 4.826367378234863,
+      "rewards/rejected": -2.637646436691284,
       "step": 215
     },
     {
-      "epoch": 1.6091743119266055,
-      "grad_norm": 12.471272777348215,
-      "learning_rate": 1.0576923076923076e-07,
-      "logits/chosen": -1.8542969226837158,
-      "logits/rejected": -1.773046851158142,
-      "logps/chosen": -201.97811889648438,
-      "logps/rejected": -332.16876220703125,
-      "loss": 0.3545,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.979199230670929,
-      "rewards/margins": 2.747509717941284,
-      "rewards/rejected": -1.767333984375,
+      "epoch": 1.3499616270145818,
+      "grad_norm": 12.767347147647387,
+      "learning_rate": 1.7313915857605177e-07,
+      "logits/chosen": -2.2425780296325684,
+      "logits/rejected": -2.064453125,
+      "logps/chosen": -268.79998779296875,
+      "logps/rejected": -459.0062561035156,
+      "loss": 0.2299,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 2.1002197265625,
+      "rewards/margins": 4.092089653015137,
+      "rewards/rejected": -1.9898560047149658,
       "step": 220
     },
     {
-      "epoch": 1.6458715596330276,
-      "grad_norm": 14.651904183676013,
-      "learning_rate": 9.615384615384616e-08,
-      "logits/chosen": -1.8474609851837158,
-      "logits/rejected": -1.7412109375,
-      "logps/chosen": -167.25,
-      "logps/rejected": -215.5812530517578,
-      "loss": 0.3734,
-      "rewards/accuracies": 0.800000011920929,
-      "rewards/chosen": 0.9526001214981079,
-      "rewards/margins": 1.8461425304412842,
-      "rewards/rejected": -0.8924926519393921,
+      "epoch": 1.3806600153491941,
+      "grad_norm": 8.681233801902874,
+      "learning_rate": 1.6504854368932038e-07,
+      "logits/chosen": -2.3003907203674316,
+      "logits/rejected": -2.0892577171325684,
+      "logps/chosen": -285.6468811035156,
+      "logps/rejected": -460.37188720703125,
+      "loss": 0.2077,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 2.2374267578125,
+      "rewards/margins": 4.366796970367432,
+      "rewards/rejected": -2.1262099742889404,
       "step": 225
     },
     {
-      "epoch": 1.6825688073394496,
-      "grad_norm": 14.445950528315446,
-      "learning_rate": 8.653846153846153e-08,
-      "logits/chosen": -1.88720703125,
-      "logits/rejected": -1.7849609851837158,
-      "logps/chosen": -163.72811889648438,
-      "logps/rejected": -273.8656311035156,
-      "loss": 0.3905,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.879486083984375,
-      "rewards/margins": 2.225268602371216,
-      "rewards/rejected": -1.3448257446289062,
+      "epoch": 1.4113584036838067,
+      "grad_norm": 10.770689073686126,
+      "learning_rate": 1.56957928802589e-07,
+      "logits/chosen": -2.287109375,
+      "logits/rejected": -2.0908203125,
+      "logps/chosen": -301.0249938964844,
+      "logps/rejected": -290.8500061035156,
+      "loss": 0.2641,
+      "rewards/accuracies": 0.893750011920929,
+      "rewards/chosen": 2.11370849609375,
+      "rewards/margins": 4.033984184265137,
+      "rewards/rejected": -1.9171874523162842,
       "step": 230
     },
     {
-      "epoch": 1.7192660550458716,
-      "grad_norm": 11.945778792987594,
-      "learning_rate": 7.692307692307692e-08,
-      "logits/chosen": -1.954492211341858,
-      "logits/rejected": -1.8380858898162842,
-      "logps/chosen": -196.75,
-      "logps/rejected": -289.890625,
-      "loss": 0.333,
-      "rewards/accuracies": 0.8374999761581421,
-      "rewards/chosen": 0.77606201171875,
-      "rewards/margins": 2.102343797683716,
-      "rewards/rejected": -1.3264648914337158,
+      "epoch": 1.442056792018419,
+      "grad_norm": 16.68254445980484,
+      "learning_rate": 1.488673139158576e-07,
+      "logits/chosen": -2.2152342796325684,
+      "logits/rejected": -2.0638670921325684,
+      "logps/chosen": -313.85626220703125,
+      "logps/rejected": -359.36248779296875,
+      "loss": 0.2247,
+      "rewards/accuracies": 0.9437500238418579,
+      "rewards/chosen": 2.3463501930236816,
+      "rewards/margins": 4.303027153015137,
+      "rewards/rejected": -1.958032250404358,
       "step": 235
     },
     {
-      "epoch": 1.7559633027522936,
-      "grad_norm": 15.570589391253572,
-      "learning_rate": 6.73076923076923e-08,
-      "logits/chosen": -1.8849608898162842,
-      "logits/rejected": -1.7628905773162842,
-      "logps/chosen": -175.00625610351562,
-      "logps/rejected": -219.3249969482422,
-      "loss": 0.327,
+      "epoch": 1.4727551803530314,
+      "grad_norm": 11.454199015230472,
+      "learning_rate": 1.407766990291262e-07,
+      "logits/chosen": -2.189453125,
+      "logits/rejected": -1.9638671875,
+      "logps/chosen": -293.0093688964844,
+      "logps/rejected": -451.98748779296875,
+      "loss": 0.2605,
       "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 0.804931640625,
-      "rewards/margins": 2.1232666969299316,
-      "rewards/rejected": -1.3185760974884033,
+      "rewards/chosen": 2.22021484375,
+      "rewards/margins": 4.625683784484863,
+      "rewards/rejected": -2.405810594558716,
       "step": 240
     },
     {
-      "epoch": 1.7926605504587156,
-      "grad_norm": 29.853484257932976,
-      "learning_rate": 5.7692307692307695e-08,
-      "logits/chosen": -1.826757788658142,
-      "logits/rejected": -1.7771484851837158,
-      "logps/chosen": -270.60467529296875,
-      "logps/rejected": -215.1843719482422,
-      "loss": 0.3607,
+      "epoch": 1.503453568687644,
+      "grad_norm": 10.811336406980383,
+      "learning_rate": 1.3268608414239483e-07,
+      "logits/chosen": -2.2994141578674316,
+      "logits/rejected": -2.1328125,
+      "logps/chosen": -321.2124938964844,
+      "logps/rejected": -442.8812561035156,
+      "loss": 0.2724,
       "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 0.9619140625,
-      "rewards/margins": 2.0130372047424316,
-      "rewards/rejected": -1.0504271984100342,
+      "rewards/chosen": 2.0484375953674316,
+      "rewards/margins": 4.695569038391113,
+      "rewards/rejected": -2.64874267578125,
       "step": 245
     },
     {
-      "epoch": 1.8293577981651377,
-      "grad_norm": 18.077930531222716,
-      "learning_rate": 4.807692307692308e-08,
-      "logits/chosen": -1.881250023841858,
-      "logits/rejected": -1.826562523841858,
-      "logps/chosen": -196.35311889648438,
-      "logps/rejected": -277.0562438964844,
-      "loss": 0.3781,
-      "rewards/accuracies": 0.831250011920929,
-      "rewards/chosen": 0.970629870891571,
-      "rewards/margins": 2.069580078125,
-      "rewards/rejected": -1.1002318859100342,
+      "epoch": 1.5341519570222564,
+      "grad_norm": 37.3725127429148,
+      "learning_rate": 1.2459546925566344e-07,
+      "logits/chosen": -2.2509765625,
+      "logits/rejected": -2.047656297683716,
+      "logps/chosen": -309.80938720703125,
+      "logps/rejected": -368.5718688964844,
+      "loss": 0.3105,
+      "rewards/accuracies": 0.862500011920929,
+      "rewards/chosen": 1.801855444908142,
+      "rewards/margins": 3.7202117443084717,
+      "rewards/rejected": -1.917608618736267,
       "step": 250
     },
     {
-      "epoch": 1.8660550458715597,
-      "grad_norm": 13.62739433233727,
-      "learning_rate": 3.846153846153846e-08,
-      "logits/chosen": -1.826757788658142,
-      "logits/rejected": -1.7664062976837158,
-      "logps/chosen": -138.7578125,
-      "logps/rejected": -210.30313110351562,
-      "loss": 0.4098,
-      "rewards/accuracies": 0.8187500238418579,
-      "rewards/chosen": 0.861804187297821,
-      "rewards/margins": 1.5693480968475342,
-      "rewards/rejected": -0.706738293170929,
+      "epoch": 1.5648503453568687,
+      "grad_norm": 12.486658586886827,
+      "learning_rate": 1.1650485436893204e-07,
+      "logits/chosen": -2.1792969703674316,
+      "logits/rejected": -2.007031202316284,
+      "logps/chosen": -275.83282470703125,
+      "logps/rejected": -367.39373779296875,
+      "loss": 0.2105,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 2.1904053688049316,
+      "rewards/margins": 4.875097751617432,
+      "rewards/rejected": -2.684741258621216,
       "step": 255
     },
     {
-      "epoch": 1.9027522935779817,
-      "grad_norm": 12.688517830934348,
-      "learning_rate": 2.8846153846153848e-08,
-      "logits/chosen": -1.883203148841858,
-      "logits/rejected": -1.759765625,
-      "logps/chosen": -167.63125610351562,
-      "logps/rejected": -209.3625030517578,
-      "loss": 0.3561,
-      "rewards/accuracies": 0.8500000238418579,
-      "rewards/chosen": 0.8629516363143921,
-      "rewards/margins": 1.875878930091858,
-      "rewards/rejected": -1.0120147466659546,
+      "epoch": 1.5955487336914813,
+      "grad_norm": 8.933287783732904,
+      "learning_rate": 1.0841423948220065e-07,
+      "logits/chosen": -2.3095703125,
+      "logits/rejected": -2.070117235183716,
+      "logps/chosen": -303.9671936035156,
+      "logps/rejected": -401.88751220703125,
+      "loss": 0.2192,
+      "rewards/accuracies": 0.918749988079071,
+      "rewards/chosen": 2.139721632003784,
+      "rewards/margins": 4.379004001617432,
+      "rewards/rejected": -2.2349791526794434,
       "step": 260
     },
     {
-      "epoch": 1.9394495412844037,
-      "grad_norm": 45.414007378020266,
-      "learning_rate": 1.923076923076923e-08,
-      "logits/chosen": -1.849023461341858,
-      "logits/rejected": -1.7927734851837158,
-      "logps/chosen": -339.13751220703125,
-      "logps/rejected": -384.7906188964844,
-      "loss": 0.4754,
-      "rewards/accuracies": 0.7749999761581421,
-      "rewards/chosen": 0.6140381097793579,
-      "rewards/margins": 2.0094971656799316,
-      "rewards/rejected": -1.395416259765625,
+      "epoch": 1.6262471220260937,
+      "grad_norm": 12.093053772689885,
+      "learning_rate": 1.0032362459546925e-07,
+      "logits/chosen": -2.1998047828674316,
+      "logits/rejected": -1.9921875,
+      "logps/chosen": -253.14688110351562,
+      "logps/rejected": -324.20623779296875,
+      "loss": 0.235,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 1.9906494617462158,
+      "rewards/margins": 4.438378810882568,
+      "rewards/rejected": -2.449389696121216,
       "step": 265
     },
     {
-      "epoch": 1.9761467889908257,
-      "grad_norm": 33.400762277633376,
-      "learning_rate": 9.615384615384615e-09,
-      "logits/chosen": -1.873437523841858,
-      "logits/rejected": -1.7628905773162842,
-      "logps/chosen": -183.55624389648438,
-      "logps/rejected": -175.9093780517578,
-      "loss": 0.4154,
-      "rewards/accuracies": 0.8812500238418579,
-      "rewards/chosen": 0.709582507610321,
-      "rewards/margins": 1.754003882408142,
-      "rewards/rejected": -1.044396996498108,
+      "epoch": 1.656945510360706,
+      "grad_norm": 23.45873113976805,
+      "learning_rate": 9.223300970873786e-08,
+      "logits/chosen": -2.155468702316284,
+      "logits/rejected": -1.975000023841858,
+      "logps/chosen": -299.0093688964844,
+      "logps/rejected": -490.6187438964844,
+      "loss": 0.2641,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 2.007080078125,
+      "rewards/margins": 4.43603515625,
+      "rewards/rejected": -2.4329833984375,
       "step": 270
     },
+    {
+      "epoch": 1.6876438986953186,
+      "grad_norm": 23.008683354113384,
+      "learning_rate": 8.414239482200647e-08,
+      "logits/chosen": -2.315234422683716,
+      "logits/rejected": -2.075000047683716,
+      "logps/chosen": -286.8187561035156,
+      "logps/rejected": -487.60626220703125,
+      "loss": 0.2542,
+      "rewards/accuracies": 0.9312499761581421,
+      "rewards/chosen": 1.4550018310546875,
+      "rewards/margins": 4.000781059265137,
+      "rewards/rejected": -2.544116258621216,
+      "step": 275
+    },
+    {
+      "epoch": 1.718342287029931,
+      "grad_norm": 9.981625176059863,
+      "learning_rate": 7.605177993527507e-08,
+      "logits/chosen": -2.2710938453674316,
+      "logits/rejected": -1.9931640625,
+      "logps/chosen": -285.4375,
+      "logps/rejected": -408.98748779296875,
+      "loss": 0.2375,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 2.16796875,
+      "rewards/margins": 4.263037204742432,
+      "rewards/rejected": -2.098034620285034,
+      "step": 280
+    },
+    {
+      "epoch": 1.7490406753645433,
+      "grad_norm": 16.26545065787498,
+      "learning_rate": 6.796116504854368e-08,
+      "logits/chosen": -2.2212891578674316,
+      "logits/rejected": -2.076367139816284,
+      "logps/chosen": -258.25701904296875,
+      "logps/rejected": -434.1187438964844,
+      "loss": 0.2327,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 2.1130614280700684,
+      "rewards/margins": 4.233984470367432,
+      "rewards/rejected": -2.1209349632263184,
+      "step": 285
+    },
+    {
+      "epoch": 1.779739063699156,
+      "grad_norm": 27.371726210067475,
+      "learning_rate": 5.987055016181229e-08,
+      "logits/chosen": -2.241406202316284,
+      "logits/rejected": -2.0464844703674316,
+      "logps/chosen": -269.08123779296875,
+      "logps/rejected": -387.34375,
+      "loss": 0.1835,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": 2.199023485183716,
+      "rewards/margins": 4.75390625,
+      "rewards/rejected": -2.554003953933716,
+      "step": 290
+    },
+    {
+      "epoch": 1.8104374520337683,
+      "grad_norm": 15.812358238339476,
+      "learning_rate": 5.1779935275080905e-08,
+      "logits/chosen": -2.2125000953674316,
+      "logits/rejected": -2.0335936546325684,
+      "logps/chosen": -239.88125610351562,
+      "logps/rejected": -359.8125,
+      "loss": 0.2308,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 1.8656494617462158,
+      "rewards/margins": 4.216113090515137,
+      "rewards/rejected": -2.351318359375,
+      "step": 295
+    },
+    {
+      "epoch": 1.8411358403683806,
+      "grad_norm": 7.50562504957695,
+      "learning_rate": 4.3689320388349516e-08,
+      "logits/chosen": -2.2437500953674316,
+      "logits/rejected": -1.9416015148162842,
+      "logps/chosen": -295.25311279296875,
+      "logps/rejected": -730.5250244140625,
+      "loss": 0.2539,
+      "rewards/accuracies": 0.90625,
+      "rewards/chosen": 2.0682129859924316,
+      "rewards/margins": 4.706347465515137,
+      "rewards/rejected": -2.636059522628784,
+      "step": 300
+    },
+    {
+      "epoch": 1.8718342287029932,
+      "grad_norm": 16.624759938399098,
+      "learning_rate": 3.559870550161812e-08,
+      "logits/chosen": -2.231250047683716,
+      "logits/rejected": -2.038867235183716,
+      "logps/chosen": -302.42498779296875,
+      "logps/rejected": -507.2562561035156,
+      "loss": 0.2846,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 1.7378661632537842,
+      "rewards/margins": 4.2021484375,
+      "rewards/rejected": -2.4639649391174316,
+      "step": 305
+    },
+    {
+      "epoch": 1.9025326170376056,
+      "grad_norm": 13.017575528579666,
+      "learning_rate": 2.750809061488673e-08,
+      "logits/chosen": -2.2279295921325684,
+      "logits/rejected": -1.9718749523162842,
+      "logps/chosen": -267.7437438964844,
+      "logps/rejected": -451.09844970703125,
+      "loss": 0.265,
+      "rewards/accuracies": 0.9125000238418579,
+      "rewards/chosen": 1.897705078125,
+      "rewards/margins": 4.704199314117432,
+      "rewards/rejected": -2.805896043777466,
+      "step": 310
+    },
+    {
+      "epoch": 1.933231005372218,
+      "grad_norm": 14.759142969719154,
+      "learning_rate": 1.9417475728155338e-08,
+      "logits/chosen": -2.226367235183716,
+      "logits/rejected": -2.0074219703674316,
+      "logps/chosen": -287.78125,
+      "logps/rejected": -434.41876220703125,
+      "loss": 0.2342,
+      "rewards/accuracies": 0.8812500238418579,
+      "rewards/chosen": 1.9349365234375,
+      "rewards/margins": 4.436230659484863,
+      "rewards/rejected": -2.49725341796875,
+      "step": 315
+    },
+    {
+      "epoch": 1.9639293937068305,
+      "grad_norm": 5.502511576422595,
+      "learning_rate": 1.1326860841423949e-08,
+      "logits/chosen": -2.2650389671325684,
+      "logits/rejected": -2.0667967796325684,
+      "logps/chosen": -285.3687438964844,
+      "logps/rejected": -342.23126220703125,
+      "loss": 0.1939,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": 2.207592725753784,
+      "rewards/margins": 4.591406345367432,
+      "rewards/rejected": -2.382006883621216,
+      "step": 320
+    },
+    {
+      "epoch": 1.9946277820414429,
+      "grad_norm": 22.06160763333907,
+      "learning_rate": 3.2362459546925565e-09,
+      "logits/chosen": -2.13671875,
+      "logits/rejected": -2.0425782203674316,
+      "logps/chosen": -295.52813720703125,
+      "logps/rejected": -390.890625,
+      "loss": 0.2874,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": 1.939697265625,
+      "rewards/margins": 3.943554639816284,
+      "rewards/rejected": -2.001660108566284,
+      "step": 325
+    },
     {
       "epoch": 2.0,
-      "step": 274,
+      "step": 326,
       "total_flos": 0.0,
-      "train_loss": 0.4061378458120527,
-      "train_runtime": 4146.7376,
-      "train_samples_per_second": 2.102,
-      "train_steps_per_second": 0.066
+      "train_loss": 0.2737119815832267,
+      "train_runtime": 3862.0227,
+      "train_samples_per_second": 2.699,
+      "train_steps_per_second": 0.084
     }
   ],
   "logging_steps": 5,
-  "max_steps": 274,
+  "max_steps": 326,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 2,
   "save_steps": 50,

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f59e267c9f87c4b6f165fec3175a328a7f6358c64f7b7474d6d509f2f903bc12
+oid sha256:d5b27d9594e9d2be8e17f4cc2533dea3a857edbe23a5da550573190398ac4436
 size 7800