Model save

README.md

@@ -0,0 +1,58 @@
+---
+base_model: mistralai/Mistral-7B-v0.1
+tags:
+- trl
+- orpo
+- generated_from_trainer
+model-index:
+- name: anydef-orpo
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# anydef-orpo
+
+This model is a fine-tuned version of [mistralai/Mistral-7B-v0.1](https://huggingface.co/mistralai/Mistral-7B-v0.1) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 8
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- total_train_batch_size: 64
+- total_eval_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: inverse_sqrt
+- lr_scheduler_warmup_steps: 100
+- num_epochs: 3
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.39.3
+- Pytorch 2.1.2+cu121
+- Datasets 2.18.0
+- Tokenizers 0.15.2

all_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.1731818498170311,
+    "train_runtime": 10691.148,
+    "train_samples": 31470,
+    "train_samples_per_second": 8.831,
+    "train_steps_per_second": 0.138
+}

config.json

@@ -0,0 +1,26 @@
+{
+  "_name_or_path": "mistralai/Mistral-7B-v0.1",
+  "architectures": [
+    "MistralForCausalLM"
+  ],
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 14336,
+  "max_position_embeddings": 32768,
+  "model_type": "mistral",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 8,
+  "rms_norm_eps": 1e-05,
+  "rope_theta": 10000.0,
+  "sliding_window": 4096,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.39.3",
+  "use_cache": false,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.39.3"
+}

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a87011969946f5de6d3a2cec805ccfd353516a64e39b9587dae8a7d0918be8b4
+size 4943162336

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3f810db6c477e69c7c6f2c7f9251f55f259a1c155b900448fe7b29e09535ede8
+size 4999819336

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f615b2b7cf7e6e10e97fc78554e9e42f4d2007f4ee3ec22f36e973f9c20ecb63
+size 4540516344

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 14483464192
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

runs/Apr19_18-05-38_hgx1/events.out.tfevents.1713551066.hgx1.696066.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09a6539e106dff6c606989e5ab4d877ab373a39ba7db3a348ce3b81e7e6a97f4
+size 131548

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dadfd56d766715c61d2ef780a525ab43b8e6da4de6865bda3d95fdef5e134055
+size 493443

tokenizer_config.json

@@ -0,0 +1,43 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [],
+  "bos_token": "<s>",
+  "chat_template": "{% for message in messages %}\n{% if message['role'] == 'user' %}\n{{ '<|user|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'system' %}\n{{ '<|system|>\n' + message['content'] + eos_token }}\n{% elif message['role'] == 'assistant' %}\n{{ '<|assistant|>\n'  + message['content'] + eos_token }}\n{% endif %}\n{% if loop.last and add_generation_prompt %}\n{{ '<|assistant|>' }}\n{% endif %}\n{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 2048,
+  "pad_token": "</s>",
+  "sp_model_kwargs": {},
+  "spaces_between_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,8 @@
+{
+    "epoch": 3.0,
+    "train_loss": 0.1731818498170311,
+    "train_runtime": 10691.148,
+    "train_samples": 31470,
+    "train_samples_per_second": 8.831,
+    "train_steps_per_second": 0.138
+}

trainer_state.json

@@ -0,0 +1,2676 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 3.0,
+  "eval_steps": 500,
+  "global_step": 1476,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.02,
+      "grad_norm": 420.0,
+      "learning_rate": 5.000000000000001e-07,
+      "log_odds_chosen": 0.4207151532173157,
+      "log_odds_ratio": -0.5201840400695801,
+      "logits/chosen": -2.347763776779175,
+      "logits/rejected": -2.3541953563690186,
+      "logps/chosen": -1.7608455419540405,
+      "logps/rejected": -2.1218907833099365,
+      "loss": 3.9956,
+      "nll_loss": 4.183717250823975,
+      "rewards/accuracies": 0.8999999761581421,
+      "rewards/chosen": -0.08804227411746979,
+      "rewards/margins": 0.018052268773317337,
+      "rewards/rejected": -0.10609455406665802,
+      "step": 10
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 44.25,
+      "learning_rate": 1.0000000000000002e-06,
+      "log_odds_chosen": 0.5392915606498718,
+      "log_odds_ratio": -0.4769781231880188,
+      "logits/chosen": -2.866349458694458,
+      "logits/rejected": -2.866671323776245,
+      "logps/chosen": -1.5143439769744873,
+      "logps/rejected": -1.9585195779800415,
+      "loss": 2.0453,
+      "nll_loss": 1.9479516744613647,
+      "rewards/accuracies": 0.925000011920929,
+      "rewards/chosen": -0.07571719586849213,
+      "rewards/margins": 0.02220877632498741,
+      "rewards/rejected": -0.09792597591876984,
+      "step": 20
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 7.5,
+      "learning_rate": 1.5e-06,
+      "log_odds_chosen": 0.9706680178642273,
+      "log_odds_ratio": -0.3677331805229187,
+      "logits/chosen": -2.6566712856292725,
+      "logits/rejected": -2.660308361053467,
+      "logps/chosen": -0.7488349676132202,
+      "logps/rejected": -1.312819480895996,
+      "loss": 1.0008,
+      "nll_loss": 0.8921732902526855,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.03744174912571907,
+      "rewards/margins": 0.028199229389429092,
+      "rewards/rejected": -0.06564097106456757,
+      "step": 30
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 7.40625,
+      "learning_rate": 2.0000000000000003e-06,
+      "log_odds_chosen": 2.282021999359131,
+      "log_odds_ratio": -0.20370522141456604,
+      "logits/chosen": -2.1984050273895264,
+      "logits/rejected": -2.202855110168457,
+      "logps/chosen": -0.27147307991981506,
+      "logps/rejected": -1.2435765266418457,
+      "loss": 0.8107,
+      "nll_loss": 0.683063268661499,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.013573653995990753,
+      "rewards/margins": 0.04860517010092735,
+      "rewards/rejected": -0.06217882037162781,
+      "step": 40
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 6.375,
+      "learning_rate": 2.5e-06,
+      "log_odds_chosen": 2.8677659034729004,
+      "log_odds_ratio": -0.14154179394245148,
+      "logits/chosen": -2.279298782348633,
+      "logits/rejected": -2.2862045764923096,
+      "logps/chosen": -0.216520756483078,
+      "logps/rejected": -1.3591465950012207,
+      "loss": 0.7978,
+      "nll_loss": 0.828228771686554,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.01082603819668293,
+      "rewards/margins": 0.057131290435791016,
+      "rewards/rejected": -0.0679573267698288,
+      "step": 50
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 4.3125,
+      "learning_rate": 3e-06,
+      "log_odds_chosen": 3.2677154541015625,
+      "log_odds_ratio": -0.12433552742004395,
+      "logits/chosen": -2.2281601428985596,
+      "logits/rejected": -2.2347497940063477,
+      "logps/chosen": -0.20724907517433167,
+      "logps/rejected": -1.5352025032043457,
+      "loss": 0.7919,
+      "nll_loss": 0.8021117448806763,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.010362453758716583,
+      "rewards/margins": 0.06639767438173294,
+      "rewards/rejected": -0.07676012814044952,
+      "step": 60
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 4.21875,
+      "learning_rate": 3.5e-06,
+      "log_odds_chosen": 3.9659018516540527,
+      "log_odds_ratio": -0.11224009841680527,
+      "logits/chosen": -2.0779483318328857,
+      "logits/rejected": -2.0807433128356934,
+      "logps/chosen": -0.1909479945898056,
+      "logps/rejected": -1.9270519018173218,
+      "loss": 0.7197,
+      "nll_loss": 0.6386193037033081,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.00954739935696125,
+      "rewards/margins": 0.08680519461631775,
+      "rewards/rejected": -0.09635259956121445,
+      "step": 70
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 4.15625,
+      "learning_rate": 4.000000000000001e-06,
+      "log_odds_chosen": 3.9601542949676514,
+      "log_odds_ratio": -0.11061631143093109,
+      "logits/chosen": -2.1697168350219727,
+      "logits/rejected": -2.1794517040252686,
+      "logps/chosen": -0.185487762093544,
+      "logps/rejected": -1.8811283111572266,
+      "loss": 0.7033,
+      "nll_loss": 0.7545000314712524,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.009274388663470745,
+      "rewards/margins": 0.08478203415870667,
+      "rewards/rejected": -0.09405642002820969,
+      "step": 80
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 4.71875,
+      "learning_rate": 4.5e-06,
+      "log_odds_chosen": 4.599364280700684,
+      "log_odds_ratio": -0.07719282805919647,
+      "logits/chosen": -2.150642156600952,
+      "logits/rejected": -2.15342378616333,
+      "logps/chosen": -0.15149493515491486,
+      "logps/rejected": -2.1990609169006348,
+      "loss": 0.6116,
+      "nll_loss": 0.6107597947120667,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.007574746850878,
+      "rewards/margins": 0.10237829387187958,
+      "rewards/rejected": -0.10995303094387054,
+      "step": 90
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 4.53125,
+      "learning_rate": 5e-06,
+      "log_odds_chosen": 4.817965507507324,
+      "log_odds_ratio": -0.07800901681184769,
+      "logits/chosen": -2.1689393520355225,
+      "logits/rejected": -2.1756036281585693,
+      "logps/chosen": -0.1730736345052719,
+      "logps/rejected": -2.246196746826172,
+      "loss": 0.6043,
+      "nll_loss": 0.6302344799041748,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.008653681725263596,
+      "rewards/margins": 0.10365615785121918,
+      "rewards/rejected": -0.11230983585119247,
+      "step": 100
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 5.625,
+      "learning_rate": 4.767312946227961e-06,
+      "log_odds_chosen": 4.97894287109375,
+      "log_odds_ratio": -0.10877744108438492,
+      "logits/chosen": -2.2094526290893555,
+      "logits/rejected": -2.216020107269287,
+      "logps/chosen": -0.18459849059581757,
+      "logps/rejected": -2.573103904724121,
+      "loss": 0.5724,
+      "nll_loss": 0.6197156310081482,
+      "rewards/accuracies": 0.9375,
+      "rewards/chosen": -0.009229923598468304,
+      "rewards/margins": 0.11942527443170547,
+      "rewards/rejected": -0.12865519523620605,
+      "step": 110
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 5.8125,
+      "learning_rate": 4.564354645876385e-06,
+      "log_odds_chosen": 5.132683277130127,
+      "log_odds_ratio": -0.13557349145412445,
+      "logits/chosen": -2.178772449493408,
+      "logits/rejected": -2.184910774230957,
+      "logps/chosen": -0.154138445854187,
+      "logps/rejected": -2.2600817680358887,
+      "loss": 0.4748,
+      "nll_loss": 0.459114134311676,
+      "rewards/accuracies": 0.949999988079071,
+      "rewards/chosen": -0.0077069224789738655,
+      "rewards/margins": 0.10529716312885284,
+      "rewards/rejected": -0.11300408840179443,
+      "step": 120
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 5.0,
+      "learning_rate": 4.385290096535147e-06,
+      "log_odds_chosen": 5.768979072570801,
+      "log_odds_ratio": -0.08052898198366165,
+      "logits/chosen": -2.101267099380493,
+      "logits/rejected": -2.104369878768921,
+      "logps/chosen": -0.1255403310060501,
+      "logps/rejected": -2.850447416305542,
+      "loss": 0.4379,
+      "nll_loss": 0.40499457716941833,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.006277016364037991,
+      "rewards/margins": 0.13624534010887146,
+      "rewards/rejected": -0.14252236485481262,
+      "step": 130
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 6.0,
+      "learning_rate": 4.2257712736425835e-06,
+      "log_odds_chosen": 5.547473907470703,
+      "log_odds_ratio": -0.056151650846004486,
+      "logits/chosen": -2.0569746494293213,
+      "logits/rejected": -2.059227466583252,
+      "logps/chosen": -0.12779685854911804,
+      "logps/rejected": -2.394996166229248,
+      "loss": 0.4122,
+      "nll_loss": 0.33014988899230957,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.006389842834323645,
+      "rewards/margins": 0.1133599653840065,
+      "rewards/rejected": -0.11974982172250748,
+      "step": 140
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 5.21875,
+      "learning_rate": 4.082482904638631e-06,
+      "log_odds_chosen": 5.67286491394043,
+      "log_odds_ratio": -0.06435154378414154,
+      "logits/chosen": -2.166313886642456,
+      "logits/rejected": -2.1680521965026855,
+      "logps/chosen": -0.13122043013572693,
+      "logps/rejected": -2.701577663421631,
+      "loss": 0.4055,
+      "nll_loss": 0.4321494996547699,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.0065610213205218315,
+      "rewards/margins": 0.12851786613464355,
+      "rewards/rejected": -0.13507887721061707,
+      "step": 150
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 6.5625,
+      "learning_rate": 3.952847075210474e-06,
+      "log_odds_chosen": 5.581357002258301,
+      "log_odds_ratio": -0.094394750893116,
+      "logits/chosen": -2.121497392654419,
+      "logits/rejected": -2.1272549629211426,
+      "logps/chosen": -0.15410242974758148,
+      "logps/rejected": -2.376707077026367,
+      "loss": 0.3695,
+      "nll_loss": 0.43908724188804626,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.007705122232437134,
+      "rewards/margins": 0.1111302375793457,
+      "rewards/rejected": -0.11883536726236343,
+      "step": 160
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 6.0,
+      "learning_rate": 3.834824944236852e-06,
+      "log_odds_chosen": 6.281979560852051,
+      "log_odds_ratio": -0.059505023062229156,
+      "logits/chosen": -2.1019725799560547,
+      "logits/rejected": -2.1044390201568604,
+      "logps/chosen": -0.15183117985725403,
+      "logps/rejected": -3.044116497039795,
+      "loss": 0.3264,
+      "nll_loss": 0.3386436998844147,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.007591559551656246,
+      "rewards/margins": 0.14461426436901093,
+      "rewards/rejected": -0.15220583975315094,
+      "step": 170
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 6.3125,
+      "learning_rate": 3.72677996249965e-06,
+      "log_odds_chosen": 6.111880302429199,
+      "log_odds_ratio": -0.07030317932367325,
+      "logits/chosen": -2.0706937313079834,
+      "logits/rejected": -2.067783832550049,
+      "logps/chosen": -0.14989063143730164,
+      "logps/rejected": -3.172281265258789,
+      "loss": 0.3324,
+      "nll_loss": 0.38789159059524536,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.007494532503187656,
+      "rewards/margins": 0.15111951529979706,
+      "rewards/rejected": -0.15861406922340393,
+      "step": 180
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 6.0625,
+      "learning_rate": 3.6273812505500587e-06,
+      "log_odds_chosen": 6.445823669433594,
+      "log_odds_ratio": -0.05266156792640686,
+      "logits/chosen": -2.0448713302612305,
+      "logits/rejected": -2.0481276512145996,
+      "logps/chosen": -0.13432614505290985,
+      "logps/rejected": -2.9074063301086426,
+      "loss": 0.3063,
+      "nll_loss": 0.28705888986587524,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.006716308183968067,
+      "rewards/margins": 0.13865402340888977,
+      "rewards/rejected": -0.14537033438682556,
+      "step": 190
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 6.0,
+      "learning_rate": 3.5355339059327378e-06,
+      "log_odds_chosen": 5.775099754333496,
+      "log_odds_ratio": -0.08722616732120514,
+      "logits/chosen": -2.087759256362915,
+      "logits/rejected": -2.0876097679138184,
+      "logps/chosen": -0.11597369611263275,
+      "logps/rejected": -2.6853129863739014,
+      "loss": 0.2962,
+      "nll_loss": 0.31049779057502747,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.005798685364425182,
+      "rewards/margins": 0.12846694886684418,
+      "rewards/rejected": -0.13426566123962402,
+      "step": 200
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 5.90625,
+      "learning_rate": 3.450327796711771e-06,
+      "log_odds_chosen": 7.5723066329956055,
+      "log_odds_ratio": -0.040629759430885315,
+      "logits/chosen": -1.973818063735962,
+      "logits/rejected": -1.976941704750061,
+      "logps/chosen": -0.09340523928403854,
+      "logps/rejected": -3.5397675037384033,
+      "loss": 0.2637,
+      "nll_loss": 0.20889191329479218,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.00467026187106967,
+      "rewards/margins": 0.17231810092926025,
+      "rewards/rejected": -0.1769883632659912,
+      "step": 210
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 6.25,
+      "learning_rate": 3.3709993123162106e-06,
+      "log_odds_chosen": 7.618361473083496,
+      "log_odds_ratio": -0.074308380484581,
+      "logits/chosen": -2.0433554649353027,
+      "logits/rejected": -2.0466837882995605,
+      "logps/chosen": -0.09071488678455353,
+      "logps/rejected": -3.5269036293029785,
+      "loss": 0.2727,
+      "nll_loss": 0.24802179634571075,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.004535744432359934,
+      "rewards/margins": 0.17180944979190826,
+      "rewards/rejected": -0.17634519934654236,
+      "step": 220
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 5.875,
+      "learning_rate": 3.296902366978936e-06,
+      "log_odds_chosen": 6.698166847229004,
+      "log_odds_ratio": -0.04654809832572937,
+      "logits/chosen": -1.968505620956421,
+      "logits/rejected": -1.9712941646575928,
+      "logps/chosen": -0.1543513834476471,
+      "logps/rejected": -3.186583995819092,
+      "loss": 0.2015,
+      "nll_loss": 0.1845930814743042,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.00771756935864687,
+      "rewards/margins": 0.15161164104938507,
+      "rewards/rejected": -0.15932922065258026,
+      "step": 230
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 5.28125,
+      "learning_rate": 3.2274861218395142e-06,
+      "log_odds_chosen": 7.480630397796631,
+      "log_odds_ratio": -0.06602514535188675,
+      "logits/chosen": -1.995495080947876,
+      "logits/rejected": -1.9951270818710327,
+      "logps/chosen": -0.12399391829967499,
+      "logps/rejected": -3.7262351512908936,
+      "loss": 0.2181,
+      "nll_loss": 0.1835182011127472,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.006199696101248264,
+      "rewards/margins": 0.1801120638847351,
+      "rewards/rejected": -0.1863117516040802,
+      "step": 240
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 6.65625,
+      "learning_rate": 3.1622776601683796e-06,
+      "log_odds_chosen": 7.167886257171631,
+      "log_odds_ratio": -0.020891310647130013,
+      "logits/chosen": -1.9453086853027344,
+      "logits/rejected": -1.9443309307098389,
+      "logps/chosen": -0.07993815094232559,
+      "logps/rejected": -3.2065398693084717,
+      "loss": 0.1966,
+      "nll_loss": 0.21496787667274475,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003996907267719507,
+      "rewards/margins": 0.15633007884025574,
+      "rewards/rejected": -0.1603269875049591,
+      "step": 250
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 7.03125,
+      "learning_rate": 3.1008683647302113e-06,
+      "log_odds_chosen": 6.189971446990967,
+      "log_odds_ratio": -0.06337615847587585,
+      "logits/chosen": -1.983068823814392,
+      "logits/rejected": -1.9870706796646118,
+      "logps/chosen": -0.13599030673503876,
+      "logps/rejected": -2.809053897857666,
+      "loss": 0.1981,
+      "nll_loss": 0.23529119789600372,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0067995162680745125,
+      "rewards/margins": 0.1336531937122345,
+      "rewards/rejected": -0.14045271277427673,
+      "step": 260
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 5.125,
+      "learning_rate": 3.0429030972509227e-06,
+      "log_odds_chosen": 6.417479038238525,
+      "log_odds_ratio": -0.05179642513394356,
+      "logits/chosen": -1.9953663349151611,
+      "logits/rejected": -1.9997583627700806,
+      "logps/chosen": -0.12870252132415771,
+      "logps/rejected": -2.9075326919555664,
+      "loss": 0.1753,
+      "nll_loss": 0.22951626777648926,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.006435126066207886,
+      "rewards/margins": 0.13894154131412506,
+      "rewards/rejected": -0.14537665247917175,
+      "step": 270
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 5.90625,
+      "learning_rate": 2.988071523335984e-06,
+      "log_odds_chosen": 6.808651924133301,
+      "log_odds_ratio": -0.06405957788228989,
+      "logits/chosen": -2.0137691497802734,
+      "logits/rejected": -2.015270709991455,
+      "logps/chosen": -0.11106196790933609,
+      "logps/rejected": -3.0401816368103027,
+      "loss": 0.1885,
+      "nll_loss": 0.1923011988401413,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.005553097929805517,
+      "rewards/margins": 0.14645597338676453,
+      "rewards/rejected": -0.15200909972190857,
+      "step": 280
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 5.375,
+      "learning_rate": 2.9361010975735177e-06,
+      "log_odds_chosen": 8.074934959411621,
+      "log_odds_ratio": -0.022188549861311913,
+      "logits/chosen": -1.94329833984375,
+      "logits/rejected": -1.943985939025879,
+      "logps/chosen": -0.06887535750865936,
+      "logps/rejected": -3.7694950103759766,
+      "loss": 0.1729,
+      "nll_loss": 0.15832534432411194,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0034437670838087797,
+      "rewards/margins": 0.185030996799469,
+      "rewards/rejected": -0.18847474455833435,
+      "step": 290
+    },
+    {
+      "epoch": 0.61,
+      "grad_norm": 4.875,
+      "learning_rate": 2.8867513459481293e-06,
+      "log_odds_chosen": 6.823982238769531,
+      "log_odds_ratio": -0.04056983441114426,
+      "logits/chosen": -1.9236243963241577,
+      "logits/rejected": -1.9263416528701782,
+      "logps/chosen": -0.15296845138072968,
+      "logps/rejected": -3.20212984085083,
+      "loss": 0.1766,
+      "nll_loss": 0.1584191620349884,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0076484219171106815,
+      "rewards/margins": 0.152458056807518,
+      "rewards/rejected": -0.16010649502277374,
+      "step": 300
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 5.0,
+      "learning_rate": 2.839809171235324e-06,
+      "log_odds_chosen": 7.289086818695068,
+      "log_odds_ratio": -0.07533205300569534,
+      "logits/chosen": -2.0198066234588623,
+      "logits/rejected": -2.022125720977783,
+      "logps/chosen": -0.12243582308292389,
+      "logps/rejected": -3.1500906944274902,
+      "loss": 0.1636,
+      "nll_loss": 0.1975417137145996,
+      "rewards/accuracies": 0.9624999761581421,
+      "rewards/chosen": -0.0061217909678816795,
+      "rewards/margins": 0.15138274431228638,
+      "rewards/rejected": -0.15750452876091003,
+      "step": 310
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 9.0,
+      "learning_rate": 2.7950849718747376e-06,
+      "log_odds_chosen": 6.9169487953186035,
+      "log_odds_ratio": -0.03769087791442871,
+      "logits/chosen": -1.9343087673187256,
+      "logits/rejected": -1.9343109130859375,
+      "logps/chosen": -0.10367435216903687,
+      "logps/rejected": -3.0478665828704834,
+      "loss": 0.1696,
+      "nll_loss": 0.15094847977161407,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.005183717701584101,
+      "rewards/margins": 0.14720964431762695,
+      "rewards/rejected": -0.15239334106445312,
+      "step": 320
+    },
+    {
+      "epoch": 0.67,
+      "grad_norm": 5.90625,
+      "learning_rate": 2.752409412815902e-06,
+      "log_odds_chosen": 6.86618185043335,
+      "log_odds_ratio": -0.038672395050525665,
+      "logits/chosen": -2.047116279602051,
+      "logits/rejected": -2.044132947921753,
+      "logps/chosen": -0.13211274147033691,
+      "logps/rejected": -3.158360004425049,
+      "loss": 0.1625,
+      "nll_loss": 0.17262741923332214,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.006605637725442648,
+      "rewards/margins": 0.15131238102912903,
+      "rewards/rejected": -0.15791800618171692,
+      "step": 330
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 5.40625,
+      "learning_rate": 2.711630722733202e-06,
+      "log_odds_chosen": 7.859139442443848,
+      "log_odds_ratio": -0.05241536349058151,
+      "logits/chosen": -1.951807975769043,
+      "logits/rejected": -1.9466063976287842,
+      "logps/chosen": -0.08200397342443466,
+      "logps/rejected": -3.5420470237731934,
+      "loss": 0.1465,
+      "nll_loss": 0.1444871723651886,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.004100198391824961,
+      "rewards/margins": 0.17300216853618622,
+      "rewards/rejected": -0.17710235714912415,
+      "step": 340
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 5.71875,
+      "learning_rate": 2.6726124191242444e-06,
+      "log_odds_chosen": 7.041175842285156,
+      "log_odds_ratio": -0.03843230381608009,
+      "logits/chosen": -1.9341367483139038,
+      "logits/rejected": -1.9377237558364868,
+      "logps/chosen": -0.105894073843956,
+      "logps/rejected": -2.9890971183776855,
+      "loss": 0.145,
+      "nll_loss": 0.15412884950637817,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.005294703878462315,
+      "rewards/margins": 0.14416013658046722,
+      "rewards/rejected": -0.14945486187934875,
+      "step": 350
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 6.90625,
+      "learning_rate": 2.6352313834736496e-06,
+      "log_odds_chosen": 6.8190507888793945,
+      "log_odds_ratio": -0.04403644800186157,
+      "logits/chosen": -1.984197974205017,
+      "logits/rejected": -1.987479567527771,
+      "logps/chosen": -0.11883167922496796,
+      "logps/rejected": -3.1214818954467773,
+      "loss": 0.1577,
+      "nll_loss": 0.12197007238864899,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0059415847063064575,
+      "rewards/margins": 0.15013250708580017,
+      "rewards/rejected": -0.15607409179210663,
+      "step": 360
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 5.28125,
+      "learning_rate": 2.599376224550182e-06,
+      "log_odds_chosen": 6.533886909484863,
+      "log_odds_ratio": -0.03479010984301567,
+      "logits/chosen": -1.898244857788086,
+      "logits/rejected": -1.8991117477416992,
+      "logps/chosen": -0.10942695289850235,
+      "logps/rejected": -2.946617841720581,
+      "loss": 0.1199,
+      "nll_loss": 0.11430631577968597,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.005471347831189632,
+      "rewards/margins": 0.14185954630374908,
+      "rewards/rejected": -0.14733090996742249,
+      "step": 370
+    },
+    {
+      "epoch": 0.77,
+      "grad_norm": 5.6875,
+      "learning_rate": 2.564945880212886e-06,
+      "log_odds_chosen": 6.550747871398926,
+      "log_odds_ratio": -0.0334211066365242,
+      "logits/chosen": -1.9049135446548462,
+      "logits/rejected": -1.9040100574493408,
+      "logps/chosen": -0.12769648432731628,
+      "logps/rejected": -3.1679463386535645,
+      "loss": 0.1321,
+      "nll_loss": 0.12015843391418457,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.006384824402630329,
+      "rewards/margins": 0.15201251208782196,
+      "rewards/rejected": -0.15839733183383942,
+      "step": 380
+    },
+    {
+      "epoch": 0.79,
+      "grad_norm": 4.46875,
+      "learning_rate": 2.5318484177091667e-06,
+      "log_odds_chosen": 6.963167667388916,
+      "log_odds_ratio": -0.07767531275749207,
+      "logits/chosen": -1.9564456939697266,
+      "logits/rejected": -1.957360029220581,
+      "logps/chosen": -0.09807981550693512,
+      "logps/rejected": -2.8958752155303955,
+      "loss": 0.1334,
+      "nll_loss": 0.13979199528694153,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.004903990775346756,
+      "rewards/margins": 0.13988977670669556,
+      "rewards/rejected": -0.1447937786579132,
+      "step": 390
+    },
+    {
+      "epoch": 0.81,
+      "grad_norm": 6.46875,
+      "learning_rate": 2.5e-06,
+      "log_odds_chosen": 7.737925052642822,
+      "log_odds_ratio": -0.011308287270367146,
+      "logits/chosen": -1.8833544254302979,
+      "logits/rejected": -1.8816306591033936,
+      "logps/chosen": -0.07808590680360794,
+      "logps/rejected": -3.517468214035034,
+      "loss": 0.116,
+      "nll_loss": 0.09308944642543793,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0039042953867465258,
+      "rewards/margins": 0.17196913063526154,
+      "rewards/rejected": -0.17587339878082275,
+      "step": 400
+    },
+    {
+      "epoch": 0.83,
+      "grad_norm": 5.09375,
+      "learning_rate": 2.4693239916239746e-06,
+      "log_odds_chosen": 8.026070594787598,
+      "log_odds_ratio": -0.009079935029149055,
+      "logits/chosen": -1.8916471004486084,
+      "logits/rejected": -1.8905131816864014,
+      "logps/chosen": -0.08094353973865509,
+      "logps/rejected": -3.705895185470581,
+      "loss": 0.1292,
+      "nll_loss": 0.10612641274929047,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.00404717680066824,
+      "rewards/margins": 0.18124757707118988,
+      "rewards/rejected": -0.1852947473526001,
+      "step": 410
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 4.15625,
+      "learning_rate": 2.4397501823713327e-06,
+      "log_odds_chosen": 6.706048488616943,
+      "log_odds_ratio": -0.04705270379781723,
+      "logits/chosen": -1.9574733972549438,
+      "logits/rejected": -1.9553906917572021,
+      "logps/chosen": -0.13810355961322784,
+      "logps/rejected": -3.244007110595703,
+      "loss": 0.1116,
+      "nll_loss": 0.13714413344860077,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.00690517807379365,
+      "rewards/margins": 0.15529517829418182,
+      "rewards/rejected": -0.16220036149024963,
+      "step": 420
+    },
+    {
+      "epoch": 0.87,
+      "grad_norm": 4.9375,
+      "learning_rate": 2.411214110852061e-06,
+      "log_odds_chosen": 7.185335636138916,
+      "log_odds_ratio": -0.049024853855371475,
+      "logits/chosen": -1.9474496841430664,
+      "logits/rejected": -1.94386887550354,
+      "logps/chosen": -0.12190796434879303,
+      "logps/rejected": -3.319507122039795,
+      "loss": 0.1158,
+      "nll_loss": 0.13795073330402374,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0060953982174396515,
+      "rewards/margins": 0.15987998247146606,
+      "rewards/rejected": -0.16597537696361542,
+      "step": 430
+    },
+    {
+      "epoch": 0.89,
+      "grad_norm": 4.8125,
+      "learning_rate": 2.3836564731139807e-06,
+      "log_odds_chosen": 7.590296268463135,
+      "log_odds_ratio": -0.021134180948138237,
+      "logits/chosen": -1.8886020183563232,
+      "logits/rejected": -1.886425256729126,
+      "logps/chosen": -0.09185922890901566,
+      "logps/rejected": -3.507038116455078,
+      "loss": 0.1033,
+      "nll_loss": 0.06822694092988968,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.00459296116605401,
+      "rewards/margins": 0.1707589477300644,
+      "rewards/rejected": -0.17535191774368286,
+      "step": 440
+    },
+    {
+      "epoch": 0.91,
+      "grad_norm": 6.28125,
+      "learning_rate": 2.357022603955159e-06,
+      "log_odds_chosen": 6.999331474304199,
+      "log_odds_ratio": -0.03660160303115845,
+      "logits/chosen": -1.861541748046875,
+      "logits/rejected": -1.861534833908081,
+      "logps/chosen": -0.12671752274036407,
+      "logps/rejected": -3.262852430343628,
+      "loss": 0.1048,
+      "nll_loss": 0.09448351711034775,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.0063358754850924015,
+      "rewards/margins": 0.15680675208568573,
+      "rewards/rejected": -0.1631426364183426,
+      "step": 450
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 4.78125,
+      "learning_rate": 2.3312620206007847e-06,
+      "log_odds_chosen": 6.333981037139893,
+      "log_odds_ratio": -0.049268852919340134,
+      "logits/chosen": -1.8751580715179443,
+      "logits/rejected": -1.8740007877349854,
+      "logps/chosen": -0.13210389018058777,
+      "logps/rejected": -2.7039647102355957,
+      "loss": 0.1094,
+      "nll_loss": 0.09672755748033524,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.006605193950235844,
+      "rewards/margins": 0.12859304249286652,
+      "rewards/rejected": -0.13519823551177979,
+      "step": 460
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 6.65625,
+      "learning_rate": 2.3063280200722128e-06,
+      "log_odds_chosen": 7.438237190246582,
+      "log_odds_ratio": -0.02831345424056053,
+      "logits/chosen": -1.9090907573699951,
+      "logits/rejected": -1.9076954126358032,
+      "logps/chosen": -0.07157810777425766,
+      "logps/rejected": -3.022064447402954,
+      "loss": 0.0996,
+      "nll_loss": 0.09075023233890533,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003578905714675784,
+      "rewards/margins": 0.14752431213855743,
+      "rewards/rejected": -0.151103213429451,
+      "step": 470
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 5.46875,
+      "learning_rate": 2.2821773229381924e-06,
+      "log_odds_chosen": 8.533849716186523,
+      "log_odds_ratio": -0.010285028256475925,
+      "logits/chosen": -1.958752989768982,
+      "logits/rejected": -1.961133599281311,
+      "logps/chosen": -0.07796324789524078,
+      "logps/rejected": -3.159803628921509,
+      "loss": 0.1074,
+      "nll_loss": 0.10170219838619232,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0038981616962701082,
+      "rewards/margins": 0.15409204363822937,
+      "rewards/rejected": -0.15799018740653992,
+      "step": 480
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 4.40625,
+      "learning_rate": 2.2587697572631284e-06,
+      "log_odds_chosen": 7.246984958648682,
+      "log_odds_ratio": -0.03945430740714073,
+      "logits/chosen": -1.9564424753189087,
+      "logits/rejected": -1.9600261449813843,
+      "logps/chosen": -0.1045081838965416,
+      "logps/rejected": -3.074450731277466,
+      "loss": 0.1118,
+      "nll_loss": 0.11936734616756439,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.00522540882229805,
+      "rewards/margins": 0.14849711954593658,
+      "rewards/rejected": -0.15372253954410553,
+      "step": 490
+    },
+    {
+      "epoch": 1.02,
+      "grad_norm": 4.78125,
+      "learning_rate": 2.23606797749979e-06,
+      "log_odds_chosen": 7.313413143157959,
+      "log_odds_ratio": -0.06336325407028198,
+      "logits/chosen": -1.925005316734314,
+      "logits/rejected": -1.9246110916137695,
+      "logps/chosen": -0.06910906732082367,
+      "logps/rejected": -3.088639974594116,
+      "loss": 0.0748,
+      "nll_loss": 0.06068875640630722,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0034554533194750547,
+      "rewards/margins": 0.15097656846046448,
+      "rewards/rejected": -0.1544320285320282,
+      "step": 500
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 3.984375,
+      "learning_rate": 2.2140372138502386e-06,
+      "log_odds_chosen": 8.353876113891602,
+      "log_odds_ratio": -0.011013227514922619,
+      "logits/chosen": -1.9190175533294678,
+      "logits/rejected": -1.9186252355575562,
+      "logps/chosen": -0.060610998421907425,
+      "logps/rejected": -3.5158791542053223,
+      "loss": 0.073,
+      "nll_loss": 0.0933486819267273,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0030305504333227873,
+      "rewards/margins": 0.1727634072303772,
+      "rewards/rejected": -0.17579396069049835,
+      "step": 510
+    },
+    {
+      "epoch": 1.06,
+      "grad_norm": 4.6875,
+      "learning_rate": 2.1926450482675734e-06,
+      "log_odds_chosen": 8.398624420166016,
+      "log_odds_ratio": -0.01034296303987503,
+      "logits/chosen": -1.9005162715911865,
+      "logits/rejected": -1.8965606689453125,
+      "logps/chosen": -0.08098746836185455,
+      "logps/rejected": -3.6455941200256348,
+      "loss": 0.0753,
+      "nll_loss": 0.0635688304901123,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.00404937332496047,
+      "rewards/margins": 0.17823031544685364,
+      "rewards/rejected": -0.18227970600128174,
+      "step": 520
+    },
+    {
+      "epoch": 1.08,
+      "grad_norm": 5.4375,
+      "learning_rate": 2.1718612138153473e-06,
+      "log_odds_chosen": 8.60987663269043,
+      "log_odds_ratio": -0.00940676499158144,
+      "logits/chosen": -1.8967382907867432,
+      "logits/rejected": -1.8937251567840576,
+      "logps/chosen": -0.07309839874505997,
+      "logps/rejected": -3.786154270172119,
+      "loss": 0.0827,
+      "nll_loss": 0.07813505828380585,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003654920030385256,
+      "rewards/margins": 0.18565279245376587,
+      "rewards/rejected": -0.18930771946907043,
+      "step": 530
+    },
+    {
+      "epoch": 1.1,
+      "grad_norm": 4.34375,
+      "learning_rate": 2.151657414559676e-06,
+      "log_odds_chosen": 9.486946105957031,
+      "log_odds_ratio": -0.006830220576375723,
+      "logits/chosen": -1.8446986675262451,
+      "logits/rejected": -1.839769721031189,
+      "logps/chosen": -0.045620597898960114,
+      "logps/rejected": -4.278280258178711,
+      "loss": 0.0759,
+      "nll_loss": 0.061400435864925385,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.002281030174344778,
+      "rewards/margins": 0.21163301169872284,
+      "rewards/rejected": -0.21391403675079346,
+      "step": 540
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 3.328125,
+      "learning_rate": 2.132007163556104e-06,
+      "log_odds_chosen": 8.17650032043457,
+      "log_odds_ratio": -0.015669699758291245,
+      "logits/chosen": -1.83901047706604,
+      "logits/rejected": -1.835837960243225,
+      "logps/chosen": -0.08997470885515213,
+      "logps/rejected": -3.697235584259033,
+      "loss": 0.0766,
+      "nll_loss": 0.05432655289769173,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.004498735070228577,
+      "rewards/margins": 0.1803630292415619,
+      "rewards/rejected": -0.18486176431179047,
+      "step": 550
+    },
+    {
+      "epoch": 1.14,
+      "grad_norm": 4.75,
+      "learning_rate": 2.1128856368212917e-06,
+      "log_odds_chosen": 7.758215427398682,
+      "log_odds_ratio": -0.027782147750258446,
+      "logits/chosen": -1.887261152267456,
+      "logits/rejected": -1.8827438354492188,
+      "logps/chosen": -0.14876721799373627,
+      "logps/rejected": -3.6743240356445312,
+      "loss": 0.0828,
+      "nll_loss": 0.11095689237117767,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.007438362576067448,
+      "rewards/margins": 0.17627784609794617,
+      "rewards/rejected": -0.18371620774269104,
+      "step": 560
+    },
+    {
+      "epoch": 1.16,
+      "grad_norm": 4.5625,
+      "learning_rate": 2.0942695414584777e-06,
+      "log_odds_chosen": 8.34414291381836,
+      "log_odds_ratio": -0.021350722759962082,
+      "logits/chosen": -1.8923943042755127,
+      "logits/rejected": -1.8904056549072266,
+      "logps/chosen": -0.09869461506605148,
+      "logps/rejected": -3.8127264976501465,
+      "loss": 0.0633,
+      "nll_loss": 0.06047019362449646,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.004934730939567089,
+      "rewards/margins": 0.18570157885551453,
+      "rewards/rejected": -0.19063633680343628,
+      "step": 570
+    },
+    {
+      "epoch": 1.18,
+      "grad_norm": 4.09375,
+      "learning_rate": 2.0761369963434992e-06,
+      "log_odds_chosen": 8.360025405883789,
+      "log_odds_ratio": -0.022461308166384697,
+      "logits/chosen": -1.9567257165908813,
+      "logits/rejected": -1.9553306102752686,
+      "logps/chosen": -0.057451434433460236,
+      "logps/rejected": -3.338317394256592,
+      "loss": 0.0647,
+      "nll_loss": 0.09119249880313873,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0028725718148052692,
+      "rewards/margins": 0.1640433371067047,
+      "rewards/rejected": -0.1669158935546875,
+      "step": 580
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 4.53125,
+      "learning_rate": 2.058467423981546e-06,
+      "log_odds_chosen": 8.788118362426758,
+      "log_odds_ratio": -0.014825952239334583,
+      "logits/chosen": -2.0043745040893555,
+      "logits/rejected": -2.00369930267334,
+      "logps/chosen": -0.06359820067882538,
+      "logps/rejected": -3.9866626262664795,
+      "loss": 0.0737,
+      "nll_loss": 0.10766579210758209,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.00317991035990417,
+      "rewards/margins": 0.19615323841571808,
+      "rewards/rejected": -0.19933313131332397,
+      "step": 590
+    },
+    {
+      "epoch": 1.22,
+      "grad_norm": 4.53125,
+      "learning_rate": 2.0412414523193154e-06,
+      "log_odds_chosen": 8.787096977233887,
+      "log_odds_ratio": -0.02094932086765766,
+      "logits/chosen": -2.0005838871002197,
+      "logits/rejected": -1.9980299472808838,
+      "logps/chosen": -0.06464333832263947,
+      "logps/rejected": -3.8292534351348877,
+      "loss": 0.0665,
+      "nll_loss": 0.07510735094547272,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0032321668695658445,
+      "rewards/margins": 0.1882304847240448,
+      "rewards/rejected": -0.1914626806974411,
+      "step": 600
+    },
+    {
+      "epoch": 1.24,
+      "grad_norm": 5.0,
+      "learning_rate": 2.0244408254472904e-06,
+      "log_odds_chosen": 8.794069290161133,
+      "log_odds_ratio": -0.0068406336940824986,
+      "logits/chosen": -1.9200255870819092,
+      "logits/rejected": -1.9168468713760376,
+      "logps/chosen": -0.07637156546115875,
+      "logps/rejected": -4.043463230133057,
+      "loss": 0.0774,
+      "nll_loss": 0.06419046223163605,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.00381857855245471,
+      "rewards/margins": 0.1983545869588852,
+      "rewards/rejected": -0.2021731585264206,
+      "step": 610
+    },
+    {
+      "epoch": 1.26,
+      "grad_norm": 3.359375,
+      "learning_rate": 2.0080483222562476e-06,
+      "log_odds_chosen": 9.725520133972168,
+      "log_odds_ratio": -0.010235416702926159,
+      "logits/chosen": -1.8615529537200928,
+      "logits/rejected": -1.8584880828857422,
+      "logps/chosen": -0.07194074988365173,
+      "logps/rejected": -4.24954891204834,
+      "loss": 0.0592,
+      "nll_loss": 0.045302603393793106,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0035970378667116165,
+      "rewards/margins": 0.20888039469718933,
+      "rewards/rejected": -0.212477445602417,
+      "step": 620
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 3.703125,
+      "learning_rate": 1.9920476822239895e-06,
+      "log_odds_chosen": 8.134096145629883,
+      "log_odds_ratio": -0.02648916281759739,
+      "logits/chosen": -1.987565279006958,
+      "logits/rejected": -1.985645055770874,
+      "logps/chosen": -0.07914531975984573,
+      "logps/rejected": -3.5829226970672607,
+      "loss": 0.076,
+      "nll_loss": 0.06963127106428146,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.003957265987992287,
+      "rewards/margins": 0.17518886923789978,
+      "rewards/rejected": -0.17914614081382751,
+      "step": 630
+    },
+    {
+      "epoch": 1.3,
+      "grad_norm": 4.65625,
+      "learning_rate": 1.976423537605237e-06,
+      "log_odds_chosen": 8.905769348144531,
+      "log_odds_ratio": -0.0063996403478085995,
+      "logits/chosen": -1.8973668813705444,
+      "logits/rejected": -1.895581841468811,
+      "logps/chosen": -0.04883747547864914,
+      "logps/rejected": -3.7183055877685547,
+      "loss": 0.0691,
+      "nll_loss": 0.060373760759830475,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0024418740067631006,
+      "rewards/margins": 0.1834733784198761,
+      "rewards/rejected": -0.1859152615070343,
+      "step": 640
+    },
+    {
+      "epoch": 1.32,
+      "grad_norm": 3.671875,
+      "learning_rate": 1.961161351381841e-06,
+      "log_odds_chosen": 8.213096618652344,
+      "log_odds_ratio": -0.012629789300262928,
+      "logits/chosen": -1.8932087421417236,
+      "logits/rejected": -1.8902513980865479,
+      "logps/chosen": -0.07331077754497528,
+      "logps/rejected": -3.4578137397766113,
+      "loss": 0.0703,
+      "nll_loss": 0.060921620577573776,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0036655396688729525,
+      "rewards/margins": 0.16922515630722046,
+      "rewards/rejected": -0.17289067804813385,
+      "step": 650
+    },
+    {
+      "epoch": 1.34,
+      "grad_norm": 4.1875,
+      "learning_rate": 1.9462473604038077e-06,
+      "log_odds_chosen": 8.2732572555542,
+      "log_odds_ratio": -0.015530670061707497,
+      "logits/chosen": -1.9228805303573608,
+      "logits/rejected": -1.9188134670257568,
+      "logps/chosen": -0.08276908099651337,
+      "logps/rejected": -3.572401523590088,
+      "loss": 0.0672,
+      "nll_loss": 0.03623514249920845,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.004138454794883728,
+      "rewards/margins": 0.1744816154241562,
+      "rewards/rejected": -0.17862007021903992,
+      "step": 660
+    },
+    {
+      "epoch": 1.36,
+      "grad_norm": 2.953125,
+      "learning_rate": 1.9316685232156397e-06,
+      "log_odds_chosen": 8.805624961853027,
+      "log_odds_ratio": -0.0074573298916220665,
+      "logits/chosen": -1.8942184448242188,
+      "logits/rejected": -1.8891878128051758,
+      "logps/chosen": -0.07351969927549362,
+      "logps/rejected": -3.884014844894409,
+      "loss": 0.0656,
+      "nll_loss": 0.05570251867175102,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0036759853828698397,
+      "rewards/margins": 0.19052477180957794,
+      "rewards/rejected": -0.1942007690668106,
+      "step": 670
+    },
+    {
+      "epoch": 1.38,
+      "grad_norm": 3.328125,
+      "learning_rate": 1.917412472118426e-06,
+      "log_odds_chosen": 8.748679161071777,
+      "log_odds_ratio": -0.024869108572602272,
+      "logits/chosen": -1.9717075824737549,
+      "logits/rejected": -1.9675525426864624,
+      "logps/chosen": -0.06451079994440079,
+      "logps/rejected": -3.9762165546417236,
+      "loss": 0.073,
+      "nll_loss": 0.11092261970043182,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.003225540043786168,
+      "rewards/margins": 0.19558526575565338,
+      "rewards/rejected": -0.19881080090999603,
+      "step": 680
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 4.90625,
+      "learning_rate": 1.9034674690672024e-06,
+      "log_odds_chosen": 8.910208702087402,
+      "log_odds_ratio": -0.005920908413827419,
+      "logits/chosen": -1.8858320713043213,
+      "logits/rejected": -1.8857864141464233,
+      "logps/chosen": -0.06332524120807648,
+      "logps/rejected": -3.758251667022705,
+      "loss": 0.0613,
+      "nll_loss": 0.07212791591882706,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003166262526065111,
+      "rewards/margins": 0.18474632501602173,
+      "rewards/rejected": -0.18791258335113525,
+      "step": 690
+    },
+    {
+      "epoch": 1.42,
+      "grad_norm": 3.25,
+      "learning_rate": 1.8898223650461362e-06,
+      "log_odds_chosen": 8.65866756439209,
+      "log_odds_ratio": -0.012265409342944622,
+      "logits/chosen": -1.9048351049423218,
+      "logits/rejected": -1.9031221866607666,
+      "logps/chosen": -0.0720919519662857,
+      "logps/rejected": -3.809000015258789,
+      "loss": 0.0596,
+      "nll_loss": 0.054995644837617874,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003604597644880414,
+      "rewards/margins": 0.18684543669223785,
+      "rewards/rejected": -0.1904500126838684,
+      "step": 700
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 3.34375,
+      "learning_rate": 1.876466562602004e-06,
+      "log_odds_chosen": 8.907628059387207,
+      "log_odds_ratio": -0.01111848745495081,
+      "logits/chosen": -1.945853590965271,
+      "logits/rejected": -1.942375898361206,
+      "logps/chosen": -0.07798963785171509,
+      "logps/rejected": -4.1663408279418945,
+      "loss": 0.0662,
+      "nll_loss": 0.08023327589035034,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003899481613188982,
+      "rewards/margins": 0.20441755652427673,
+      "rewards/rejected": -0.20831704139709473,
+      "step": 710
+    },
+    {
+      "epoch": 1.46,
+      "grad_norm": 5.28125,
+      "learning_rate": 1.863389981249825e-06,
+      "log_odds_chosen": 9.416102409362793,
+      "log_odds_ratio": -0.007732940372079611,
+      "logits/chosen": -1.8631290197372437,
+      "logits/rejected": -1.8593025207519531,
+      "logps/chosen": -0.06107889488339424,
+      "logps/rejected": -4.503323554992676,
+      "loss": 0.0585,
+      "nll_loss": 0.05343944951891899,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0030539450235664845,
+      "rewards/margins": 0.22211222350597382,
+      "rewards/rejected": -0.2251661717891693,
+      "step": 720
+    },
+    {
+      "epoch": 1.48,
+      "grad_norm": 2.78125,
+      "learning_rate": 1.8505830254940132e-06,
+      "log_odds_chosen": 9.134099960327148,
+      "log_odds_ratio": -0.011219611391425133,
+      "logits/chosen": -1.9615509510040283,
+      "logits/rejected": -1.9607875347137451,
+      "logps/chosen": -0.05891376733779907,
+      "logps/rejected": -4.027497291564941,
+      "loss": 0.0582,
+      "nll_loss": 0.050385020673274994,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0029456880874931812,
+      "rewards/margins": 0.1984291821718216,
+      "rewards/rejected": -0.2013748586177826,
+      "step": 730
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 4.0625,
+      "learning_rate": 1.8380365552345197e-06,
+      "log_odds_chosen": 8.314208984375,
+      "log_odds_ratio": -0.017240628600120544,
+      "logits/chosen": -1.9306405782699585,
+      "logits/rejected": -1.9276716709136963,
+      "logps/chosen": -0.12811905145645142,
+      "logps/rejected": -3.857971668243408,
+      "loss": 0.0666,
+      "nll_loss": 0.07230392098426819,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.006405952386558056,
+      "rewards/margins": 0.18649262189865112,
+      "rewards/rejected": -0.19289858639240265,
+      "step": 740
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 6.8125,
+      "learning_rate": 1.8257418583505536e-06,
+      "log_odds_chosen": 8.118074417114258,
+      "log_odds_ratio": -0.04242696613073349,
+      "logits/chosen": -1.951964020729065,
+      "logits/rejected": -1.9483181238174438,
+      "logps/chosen": -0.07924682646989822,
+      "logps/rejected": -3.6926894187927246,
+      "loss": 0.0608,
+      "nll_loss": 0.05164622515439987,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.003962341230362654,
+      "rewards/margins": 0.18067213892936707,
+      "rewards/rejected": -0.1846344769001007,
+      "step": 750
+    },
+    {
+      "epoch": 1.54,
+      "grad_norm": 2.890625,
+      "learning_rate": 1.8136906252750293e-06,
+      "log_odds_chosen": 8.884002685546875,
+      "log_odds_ratio": -0.007163527421653271,
+      "logits/chosen": -1.8522392511367798,
+      "logits/rejected": -1.8525257110595703,
+      "logps/chosen": -0.06406822055578232,
+      "logps/rejected": -3.8873114585876465,
+      "loss": 0.0508,
+      "nll_loss": 0.052127063274383545,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003203411353752017,
+      "rewards/margins": 0.19116216897964478,
+      "rewards/rejected": -0.19436559081077576,
+      "step": 760
+    },
+    {
+      "epoch": 1.57,
+      "grad_norm": 4.8125,
+      "learning_rate": 1.801874925391118e-06,
+      "log_odds_chosen": 8.48697566986084,
+      "log_odds_ratio": -0.026209736242890358,
+      "logits/chosen": -1.9563274383544922,
+      "logits/rejected": -1.9566619396209717,
+      "logps/chosen": -0.075563944876194,
+      "logps/rejected": -3.6374893188476562,
+      "loss": 0.0665,
+      "nll_loss": 0.05961776524782181,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0037781973369419575,
+      "rewards/margins": 0.17809626460075378,
+      "rewards/rejected": -0.18187445402145386,
+      "step": 770
+    },
+    {
+      "epoch": 1.59,
+      "grad_norm": 3.703125,
+      "learning_rate": 1.7902871850985824e-06,
+      "log_odds_chosen": 9.119174003601074,
+      "log_odds_ratio": -0.024956868961453438,
+      "logits/chosen": -1.918013572692871,
+      "logits/rejected": -1.9134963750839233,
+      "logps/chosen": -0.05534970015287399,
+      "logps/rejected": -4.010322570800781,
+      "loss": 0.0595,
+      "nll_loss": 0.04103861376643181,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0027674853336066008,
+      "rewards/margins": 0.19774861633777618,
+      "rewards/rejected": -0.20051613450050354,
+      "step": 780
+    },
+    {
+      "epoch": 1.61,
+      "grad_norm": 3.6875,
+      "learning_rate": 1.7789201674120502e-06,
+      "log_odds_chosen": 8.733691215515137,
+      "log_odds_ratio": -0.008985884487628937,
+      "logits/chosen": -1.8730590343475342,
+      "logits/rejected": -1.8695628643035889,
+      "logps/chosen": -0.05646519735455513,
+      "logps/rejected": -3.503816604614258,
+      "loss": 0.0506,
+      "nll_loss": 0.047934651374816895,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.002823260147124529,
+      "rewards/margins": 0.17236760258674622,
+      "rewards/rejected": -0.17519085109233856,
+      "step": 790
+    },
+    {
+      "epoch": 1.63,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.7677669529663689e-06,
+      "log_odds_chosen": 8.943314552307129,
+      "log_odds_ratio": -0.00942598469555378,
+      "logits/chosen": -1.8948551416397095,
+      "logits/rejected": -1.892686128616333,
+      "logps/chosen": -0.05206329748034477,
+      "logps/rejected": -3.80363130569458,
+      "loss": 0.0526,
+      "nll_loss": 0.06757352501153946,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0026031648740172386,
+      "rewards/margins": 0.18757839500904083,
+      "rewards/rejected": -0.19018153846263885,
+      "step": 800
+    },
+    {
+      "epoch": 1.65,
+      "grad_norm": 3.515625,
+      "learning_rate": 1.7568209223157664e-06,
+      "log_odds_chosen": 9.602978706359863,
+      "log_odds_ratio": -0.0054091801866889,
+      "logits/chosen": -1.8749698400497437,
+      "logits/rejected": -1.8726646900177002,
+      "logps/chosen": -0.049051497131586075,
+      "logps/rejected": -4.167695045471191,
+      "loss": 0.0527,
+      "nll_loss": 0.08227180689573288,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0024525749031454325,
+      "rewards/margins": 0.20593218505382538,
+      "rewards/rejected": -0.20838478207588196,
+      "step": 810
+    },
+    {
+      "epoch": 1.67,
+      "grad_norm": 4.1875,
+      "learning_rate": 1.7460757394239458e-06,
+      "log_odds_chosen": 8.731072425842285,
+      "log_odds_ratio": -0.006553919520229101,
+      "logits/chosen": -1.9533252716064453,
+      "logits/rejected": -1.9529660940170288,
+      "logps/chosen": -0.0727374479174614,
+      "logps/rejected": -3.5265769958496094,
+      "loss": 0.0659,
+      "nll_loss": 0.0733790472149849,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003636872861534357,
+      "rewards/margins": 0.17269198596477509,
+      "rewards/rejected": -0.1763288527727127,
+      "step": 820
+    },
+    {
+      "epoch": 1.69,
+      "grad_norm": 2.984375,
+      "learning_rate": 1.7355253362515584e-06,
+      "log_odds_chosen": 9.135687828063965,
+      "log_odds_ratio": -0.03666002303361893,
+      "logits/chosen": -1.9465566873550415,
+      "logits/rejected": -1.9425933361053467,
+      "logps/chosen": -0.07385378330945969,
+      "logps/rejected": -3.989952564239502,
+      "loss": 0.0478,
+      "nll_loss": 0.06535151600837708,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.003692689584568143,
+      "rewards/margins": 0.1958049088716507,
+      "rewards/rejected": -0.19949761033058167,
+      "step": 830
+    },
+    {
+      "epoch": 1.71,
+      "grad_norm": 2.9375,
+      "learning_rate": 1.7251638983558855e-06,
+      "log_odds_chosen": 8.61412239074707,
+      "log_odds_ratio": -0.006854639388620853,
+      "logits/chosen": -1.8970463275909424,
+      "logits/rejected": -1.8957879543304443,
+      "logps/chosen": -0.09370087087154388,
+      "logps/rejected": -3.841550827026367,
+      "loss": 0.0587,
+      "nll_loss": 0.046143922954797745,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.004685043357312679,
+      "rewards/margins": 0.18739250302314758,
+      "rewards/rejected": -0.19207754731178284,
+      "step": 840
+    },
+    {
+      "epoch": 1.73,
+      "grad_norm": 3.40625,
+      "learning_rate": 1.7149858514250883e-06,
+      "log_odds_chosen": 9.044164657592773,
+      "log_odds_ratio": -0.023376554250717163,
+      "logits/chosen": -1.9251768589019775,
+      "logits/rejected": -1.9241018295288086,
+      "logps/chosen": -0.05019516497850418,
+      "logps/rejected": -3.740339994430542,
+      "loss": 0.0537,
+      "nll_loss": 0.04762560874223709,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0025097583420574665,
+      "rewards/margins": 0.18450722098350525,
+      "rewards/rejected": -0.18701700866222382,
+      "step": 850
+    },
+    {
+      "epoch": 1.75,
+      "grad_norm": 2.703125,
+      "learning_rate": 1.704985848676184e-06,
+      "log_odds_chosen": 8.560470581054688,
+      "log_odds_ratio": -0.015030615031719208,
+      "logits/chosen": -1.893930435180664,
+      "logits/rejected": -1.8902561664581299,
+      "logps/chosen": -0.07954917848110199,
+      "logps/rejected": -3.786511182785034,
+      "loss": 0.0544,
+      "nll_loss": 0.054652292281389236,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0039774589240550995,
+      "rewards/margins": 0.18534812331199646,
+      "rewards/rejected": -0.18932557106018066,
+      "step": 860
+    },
+    {
+      "epoch": 1.77,
+      "grad_norm": 2.9375,
+      "learning_rate": 1.6951587590520263e-06,
+      "log_odds_chosen": 9.6688232421875,
+      "log_odds_ratio": -0.010795338079333305,
+      "logits/chosen": -1.9225839376449585,
+      "logits/rejected": -1.9220905303955078,
+      "logps/chosen": -0.048293761909008026,
+      "logps/rejected": -4.323627948760986,
+      "loss": 0.0488,
+      "nll_loss": 0.043466441333293915,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.002414688002318144,
+      "rewards/margins": 0.21376672387123108,
+      "rewards/rejected": -0.2161814272403717,
+      "step": 870
+    },
+    {
+      "epoch": 1.79,
+      "grad_norm": 3.234375,
+      "learning_rate": 1.6854996561581053e-06,
+      "log_odds_chosen": 9.363157272338867,
+      "log_odds_ratio": -0.026759719476103783,
+      "logits/chosen": -1.9485028982162476,
+      "logits/rejected": -1.9452836513519287,
+      "logps/chosen": -0.042739786207675934,
+      "logps/rejected": -3.5607593059539795,
+      "loss": 0.0555,
+      "nll_loss": 0.04253282770514488,
+      "rewards/accuracies": 0.9750000238418579,
+      "rewards/chosen": -0.002136989263817668,
+      "rewards/margins": 0.17590096592903137,
+      "rewards/rejected": -0.17803797125816345,
+      "step": 880
+    },
+    {
+      "epoch": 1.81,
+      "grad_norm": 4.1875,
+      "learning_rate": 1.6760038078849776e-06,
+      "log_odds_chosen": 8.319189071655273,
+      "log_odds_ratio": -0.023240404203534126,
+      "logits/chosen": -1.851123571395874,
+      "logits/rejected": -1.849542260169983,
+      "logps/chosen": -0.06788856536149979,
+      "logps/rejected": -3.406628131866455,
+      "loss": 0.0603,
+      "nll_loss": 0.04571394622325897,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.003394428174942732,
+      "rewards/margins": 0.1669369637966156,
+      "rewards/rejected": -0.17033140361309052,
+      "step": 890
+    },
+    {
+      "epoch": 1.83,
+      "grad_norm": 4.21875,
+      "learning_rate": 1.6666666666666667e-06,
+      "log_odds_chosen": 8.61945915222168,
+      "log_odds_ratio": -0.017158176749944687,
+      "logits/chosen": -1.9368407726287842,
+      "logits/rejected": -1.9338048696517944,
+      "logps/chosen": -0.05057498812675476,
+      "logps/rejected": -3.4434471130371094,
+      "loss": 0.0499,
+      "nll_loss": 0.05269382521510124,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0025287498719990253,
+      "rewards/margins": 0.1696436107158661,
+      "rewards/rejected": -0.17217235267162323,
+      "step": 900
+    },
+    {
+      "epoch": 1.85,
+      "grad_norm": 3.265625,
+      "learning_rate": 1.6574838603294898e-06,
+      "log_odds_chosen": 9.762075424194336,
+      "log_odds_ratio": -0.0037377229891717434,
+      "logits/chosen": -2.0028135776519775,
+      "logits/rejected": -2.0014703273773193,
+      "logps/chosen": -0.034680210053920746,
+      "logps/rejected": -4.053231239318848,
+      "loss": 0.0576,
+      "nll_loss": 0.057746343314647675,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0017340105259791017,
+      "rewards/margins": 0.20092757046222687,
+      "rewards/rejected": -0.20266155898571014,
+      "step": 910
+    },
+    {
+      "epoch": 1.87,
+      "grad_norm": 3.0,
+      "learning_rate": 1.648451183489468e-06,
+      "log_odds_chosen": 9.181684494018555,
+      "log_odds_ratio": -0.007746308110654354,
+      "logits/chosen": -1.935484528541565,
+      "logits/rejected": -1.932363748550415,
+      "logps/chosen": -0.08534862101078033,
+      "logps/rejected": -4.044074535369873,
+      "loss": 0.0526,
+      "nll_loss": 0.06775641441345215,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.004267430864274502,
+      "rewards/margins": 0.1979363113641739,
+      "rewards/rejected": -0.20220375061035156,
+      "step": 920
+    },
+    {
+      "epoch": 1.89,
+      "grad_norm": 1.609375,
+      "learning_rate": 1.6395645894598825e-06,
+      "log_odds_chosen": 9.165306091308594,
+      "log_odds_ratio": -0.01449726801365614,
+      "logits/chosen": -1.9488122463226318,
+      "logits/rejected": -1.9472631216049194,
+      "logps/chosen": -0.07069545984268188,
+      "logps/rejected": -3.8849570751190186,
+      "loss": 0.057,
+      "nll_loss": 0.052025146782398224,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0035347728990018368,
+      "rewards/margins": 0.1907130777835846,
+      "rewards/rejected": -0.19424787163734436,
+      "step": 930
+    },
+    {
+      "epoch": 1.91,
+      "grad_norm": 3.90625,
+      "learning_rate": 1.6308201826336057e-06,
+      "log_odds_chosen": 9.020658493041992,
+      "log_odds_ratio": -0.00849425233900547,
+      "logits/chosen": -1.916486144065857,
+      "logits/rejected": -1.9109046459197998,
+      "logps/chosen": -0.06480909883975983,
+      "logps/rejected": -4.059305667877197,
+      "loss": 0.0504,
+      "nll_loss": 0.058377087116241455,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0032404554076492786,
+      "rewards/margins": 0.19972485303878784,
+      "rewards/rejected": -0.20296530425548553,
+      "step": 940
+    },
+    {
+      "epoch": 1.93,
+      "grad_norm": 3.75,
+      "learning_rate": 1.6222142113076255e-06,
+      "log_odds_chosen": 9.191986083984375,
+      "log_odds_ratio": -0.007987757213413715,
+      "logits/chosen": -1.8939701318740845,
+      "logits/rejected": -1.892139196395874,
+      "logps/chosen": -0.042362868785858154,
+      "logps/rejected": -3.922192335128784,
+      "loss": 0.0543,
+      "nll_loss": 0.0392763689160347,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0021181434858590364,
+      "rewards/margins": 0.19399148225784302,
+      "rewards/rejected": -0.1961096078157425,
+      "step": 950
+    },
+    {
+      "epoch": 1.95,
+      "grad_norm": 3.65625,
+      "learning_rate": 1.6137430609197571e-06,
+      "log_odds_chosen": 8.950517654418945,
+      "log_odds_ratio": -0.01621365174651146,
+      "logits/chosen": -1.9614036083221436,
+      "logits/rejected": -1.9606664180755615,
+      "logps/chosen": -0.07848796993494034,
+      "logps/rejected": -4.038422584533691,
+      "loss": 0.0554,
+      "nll_loss": 0.0618586540222168,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003924398683011532,
+      "rewards/margins": 0.19799675047397614,
+      "rewards/rejected": -0.20192115008831024,
+      "step": 960
+    },
+    {
+      "epoch": 1.97,
+      "grad_norm": 2.875,
+      "learning_rate": 1.605403247669839e-06,
+      "log_odds_chosen": 8.41962718963623,
+      "log_odds_ratio": -0.020061850547790527,
+      "logits/chosen": -1.908539056777954,
+      "logits/rejected": -1.9044193029403687,
+      "logps/chosen": -0.07218164205551147,
+      "logps/rejected": -3.6821017265319824,
+      "loss": 0.0528,
+      "nll_loss": 0.03915850818157196,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.003609082894399762,
+      "rewards/margins": 0.1804960072040558,
+      "rewards/rejected": -0.18410511314868927,
+      "step": 970
+    },
+    {
+      "epoch": 1.99,
+      "grad_norm": 4.4375,
+      "learning_rate": 1.59719141249985e-06,
+      "log_odds_chosen": 8.960528373718262,
+      "log_odds_ratio": -0.007182210683822632,
+      "logits/chosen": -2.059950351715088,
+      "logits/rejected": -2.058908224105835,
+      "logps/chosen": -0.07827770709991455,
+      "logps/rejected": -3.9805305004119873,
+      "loss": 0.0589,
+      "nll_loss": 0.08598147332668304,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003913884982466698,
+      "rewards/margins": 0.19511264562606812,
+      "rewards/rejected": -0.19902652502059937,
+      "step": 980
+    },
+    {
+      "epoch": 2.01,
+      "grad_norm": 2.421875,
+      "learning_rate": 1.5891043154093205e-06,
+      "log_odds_chosen": 8.75261402130127,
+      "log_odds_ratio": -0.008246051147580147,
+      "logits/chosen": -1.9300010204315186,
+      "logits/rejected": -1.9284627437591553,
+      "logps/chosen": -0.059024132788181305,
+      "logps/rejected": -3.7112643718719482,
+      "loss": 0.0501,
+      "nll_loss": 0.0379130020737648,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0029512066394090652,
+      "rewards/margins": 0.18261203169822693,
+      "rewards/rejected": -0.18556323647499084,
+      "step": 990
+    },
+    {
+      "epoch": 2.03,
+      "grad_norm": 2.4375,
+      "learning_rate": 1.5811388300841898e-06,
+      "log_odds_chosen": 9.132776260375977,
+      "log_odds_ratio": -0.010855279862880707,
+      "logits/chosen": -1.9048831462860107,
+      "logits/rejected": -1.9021356105804443,
+      "logps/chosen": -0.05851290374994278,
+      "logps/rejected": -3.9342761039733887,
+      "loss": 0.0399,
+      "nll_loss": 0.045368582010269165,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.002925645327195525,
+      "rewards/margins": 0.19378817081451416,
+      "rewards/rejected": -0.19671380519866943,
+      "step": 1000
+    },
+    {
+      "epoch": 2.05,
+      "grad_norm": 3.34375,
+      "learning_rate": 1.5732919388188816e-06,
+      "log_odds_chosen": 9.254007339477539,
+      "log_odds_ratio": -0.004844398703426123,
+      "logits/chosen": -1.906938910484314,
+      "logits/rejected": -1.9027900695800781,
+      "logps/chosen": -0.0639704093337059,
+      "logps/rejected": -4.133778095245361,
+      "loss": 0.0377,
+      "nll_loss": 0.03271421417593956,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003198520513251424,
+      "rewards/margins": 0.20349040627479553,
+      "rewards/rejected": -0.20668891072273254,
+      "step": 1010
+    },
+    {
+      "epoch": 2.07,
+      "grad_norm": 2.34375,
+      "learning_rate": 1.565560727712874e-06,
+      "log_odds_chosen": 9.79780387878418,
+      "log_odds_ratio": -0.0035797320306301117,
+      "logits/chosen": -1.934853196144104,
+      "logits/rejected": -1.9338003396987915,
+      "logps/chosen": -0.05114441365003586,
+      "logps/rejected": -4.095462799072266,
+      "loss": 0.0456,
+      "nll_loss": 0.055598996579647064,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.002557220868766308,
+      "rewards/margins": 0.20221590995788574,
+      "rewards/rejected": -0.20477311313152313,
+      "step": 1020
+    },
+    {
+      "epoch": 2.09,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.5579423821243897e-06,
+      "log_odds_chosen": 10.34232234954834,
+      "log_odds_ratio": -0.002075557829812169,
+      "logits/chosen": -1.908393144607544,
+      "logits/rejected": -1.9075828790664673,
+      "logps/chosen": -0.03475125879049301,
+      "logps/rejected": -4.130476474761963,
+      "loss": 0.0436,
+      "nll_loss": 0.04652316868305206,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0017375629395246506,
+      "rewards/margins": 0.2047862708568573,
+      "rewards/rejected": -0.2065238207578659,
+      "step": 1030
+    },
+    {
+      "epoch": 2.11,
+      "grad_norm": 3.796875,
+      "learning_rate": 1.5504341823651056e-06,
+      "log_odds_chosen": 9.534278869628906,
+      "log_odds_ratio": -0.014330191537737846,
+      "logits/chosen": -1.8937572240829468,
+      "logits/rejected": -1.8925514221191406,
+      "logps/chosen": -0.05281580239534378,
+      "logps/rejected": -3.8376965522766113,
+      "loss": 0.0506,
+      "nll_loss": 0.04847393184900284,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0026407900732010603,
+      "rewards/margins": 0.18924403190612793,
+      "rewards/rejected": -0.1918848156929016,
+      "step": 1040
+    },
+    {
+      "epoch": 2.13,
+      "grad_norm": 2.765625,
+      "learning_rate": 1.5430334996209192e-06,
+      "log_odds_chosen": 9.982309341430664,
+      "log_odds_ratio": -0.0034835089463740587,
+      "logits/chosen": -1.9073753356933594,
+      "logits/rejected": -1.9033172130584717,
+      "logps/chosen": -0.03734086453914642,
+      "logps/rejected": -4.198154449462891,
+      "loss": 0.0399,
+      "nll_loss": 0.03343256562948227,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0018670432036742568,
+      "rewards/margins": 0.20804063975811005,
+      "rewards/rejected": -0.20990769565105438,
+      "step": 1050
+    },
+    {
+      "epoch": 2.15,
+      "grad_norm": 2.015625,
+      "learning_rate": 1.5357377920848783e-06,
+      "log_odds_chosen": 9.932341575622559,
+      "log_odds_ratio": -0.00664928276091814,
+      "logits/chosen": -1.968000054359436,
+      "logits/rejected": -1.9650490283966064,
+      "logps/chosen": -0.053252361714839935,
+      "logps/rejected": -4.3154191970825195,
+      "loss": 0.0398,
+      "nll_loss": 0.03708261623978615,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0026626181788742542,
+      "rewards/margins": 0.2131083756685257,
+      "rewards/rejected": -0.21577095985412598,
+      "step": 1060
+    },
+    {
+      "epoch": 2.17,
+      "grad_norm": 3.203125,
+      "learning_rate": 1.5285446012893579e-06,
+      "log_odds_chosen": 10.412023544311523,
+      "log_odds_ratio": -0.004382356535643339,
+      "logits/chosen": -1.8630377054214478,
+      "logits/rejected": -1.858724594116211,
+      "logps/chosen": -0.03156786039471626,
+      "logps/rejected": -4.467475414276123,
+      "loss": 0.0435,
+      "nll_loss": 0.043990008533000946,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0015783928101882339,
+      "rewards/margins": 0.22179536521434784,
+      "rewards/rejected": -0.22337380051612854,
+      "step": 1070
+    },
+    {
+      "epoch": 2.2,
+      "grad_norm": 2.859375,
+      "learning_rate": 1.5214515486254614e-06,
+      "log_odds_chosen": 9.860450744628906,
+      "log_odds_ratio": -0.00421870406717062,
+      "logits/chosen": -1.9334264993667603,
+      "logits/rejected": -1.9309759140014648,
+      "logps/chosen": -0.04163174703717232,
+      "logps/rejected": -4.278031349182129,
+      "loss": 0.0399,
+      "nll_loss": 0.03083736263215542,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0020815874449908733,
+      "rewards/margins": 0.21182000637054443,
+      "rewards/rejected": -0.2139015644788742,
+      "step": 1080
+    },
+    {
+      "epoch": 2.22,
+      "grad_norm": 3.375,
+      "learning_rate": 1.5144563320384566e-06,
+      "log_odds_chosen": 9.61739444732666,
+      "log_odds_ratio": -0.007051125168800354,
+      "logits/chosen": -1.9654079675674438,
+      "logits/rejected": -1.9627532958984375,
+      "logps/chosen": -0.04155623912811279,
+      "logps/rejected": -3.9517130851745605,
+      "loss": 0.0419,
+      "nll_loss": 0.0399235263466835,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.002077811863273382,
+      "rewards/margins": 0.19550786912441254,
+      "rewards/rejected": -0.19758570194244385,
+      "step": 1090
+    },
+    {
+      "epoch": 2.24,
+      "grad_norm": 2.5625,
+      "learning_rate": 1.5075567228888182e-06,
+      "log_odds_chosen": 9.472233772277832,
+      "log_odds_ratio": -0.004241319373250008,
+      "logits/chosen": -1.8757314682006836,
+      "logits/rejected": -1.875422477722168,
+      "logps/chosen": -0.04146653413772583,
+      "logps/rejected": -3.9626433849334717,
+      "loss": 0.0388,
+      "nll_loss": 0.044396962970495224,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.002073326613754034,
+      "rewards/margins": 0.19605886936187744,
+      "rewards/rejected": -0.19813218712806702,
+      "step": 1100
+    },
+    {
+      "epoch": 2.26,
+      "grad_norm": 2.84375,
+      "learning_rate": 1.5007505629691608e-06,
+      "log_odds_chosen": 10.060752868652344,
+      "log_odds_ratio": -0.005245304200798273,
+      "logits/chosen": -1.884495735168457,
+      "logits/rejected": -1.8812463283538818,
+      "logps/chosen": -0.042206455022096634,
+      "logps/rejected": -4.164919853210449,
+      "loss": 0.0437,
+      "nll_loss": 0.03563828393816948,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.002110322704538703,
+      "rewards/margins": 0.20613567531108856,
+      "rewards/rejected": -0.20824599266052246,
+      "step": 1110
+    },
+    {
+      "epoch": 2.28,
+      "grad_norm": 1.875,
+      "learning_rate": 1.494035761667992e-06,
+      "log_odds_chosen": 10.209965705871582,
+      "log_odds_ratio": -0.00312315602786839,
+      "logits/chosen": -1.921362280845642,
+      "logits/rejected": -1.9180587530136108,
+      "logps/chosen": -0.04360593855381012,
+      "logps/rejected": -4.336488723754883,
+      "loss": 0.0425,
+      "nll_loss": 0.050070591270923615,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.002180296927690506,
+      "rewards/margins": 0.2146441489458084,
+      "rewards/rejected": -0.21682441234588623,
+      "step": 1120
+    },
+    {
+      "epoch": 2.3,
+      "grad_norm": 3.34375,
+      "learning_rate": 1.487410293271824e-06,
+      "log_odds_chosen": 9.571352005004883,
+      "log_odds_ratio": -0.0124302227050066,
+      "logits/chosen": -1.9245920181274414,
+      "logits/rejected": -1.9231822490692139,
+      "logps/chosen": -0.051782481372356415,
+      "logps/rejected": -4.130411624908447,
+      "loss": 0.042,
+      "nll_loss": 0.04519005864858627,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.002589124022051692,
+      "rewards/margins": 0.20393142104148865,
+      "rewards/rejected": -0.20652055740356445,
+      "step": 1130
+    },
+    {
+      "epoch": 2.32,
+      "grad_norm": 2.0625,
+      "learning_rate": 1.480872194397731e-06,
+      "log_odds_chosen": 9.836039543151855,
+      "log_odds_ratio": -0.004647588822990656,
+      "logits/chosen": -1.8972570896148682,
+      "logits/rejected": -1.8939294815063477,
+      "logps/chosen": -0.0661562830209732,
+      "logps/rejected": -4.237457752227783,
+      "loss": 0.0435,
+      "nll_loss": 0.04812124744057655,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0033078144770115614,
+      "rewards/margins": 0.20856507122516632,
+      "rewards/rejected": -0.211872860789299,
+      "step": 1140
+    },
+    {
+      "epoch": 2.34,
+      "grad_norm": 3.546875,
+      "learning_rate": 1.4744195615489715e-06,
+      "log_odds_chosen": 10.264336585998535,
+      "log_odds_ratio": -0.0017655363772064447,
+      "logits/chosen": -1.9511454105377197,
+      "logits/rejected": -1.9473825693130493,
+      "logps/chosen": -0.04254579171538353,
+      "logps/rejected": -4.465078353881836,
+      "loss": 0.0408,
+      "nll_loss": 0.03954671695828438,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0021272895392030478,
+      "rewards/margins": 0.22112664580345154,
+      "rewards/rejected": -0.22325392067432404,
+      "step": 1150
+    },
+    {
+      "epoch": 2.36,
+      "grad_norm": 2.203125,
+      "learning_rate": 1.4680505487867589e-06,
+      "log_odds_chosen": 9.718263626098633,
+      "log_odds_ratio": -0.002675818046554923,
+      "logits/chosen": -1.9049984216690063,
+      "logits/rejected": -1.9013592004776,
+      "logps/chosen": -0.04851164296269417,
+      "logps/rejected": -4.104137420654297,
+      "loss": 0.0396,
+      "nll_loss": 0.03321639448404312,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0024255819153040648,
+      "rewards/margins": 0.2027812898159027,
+      "rewards/rejected": -0.20520684123039246,
+      "step": 1160
+    },
+    {
+      "epoch": 2.38,
+      "grad_norm": 1.8828125,
+      "learning_rate": 1.4617633655117156e-06,
+      "log_odds_chosen": 9.280858993530273,
+      "log_odds_ratio": -0.008265355601906776,
+      "logits/chosen": -1.964536428451538,
+      "logits/rejected": -1.9617042541503906,
+      "logps/chosen": -0.06000961735844612,
+      "logps/rejected": -4.104333400726318,
+      "loss": 0.0435,
+      "nll_loss": 0.04580981656908989,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003000481054186821,
+      "rewards/margins": 0.20221619307994843,
+      "rewards/rejected": -0.2052166759967804,
+      "step": 1170
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 2.578125,
+      "learning_rate": 1.4555562743489552e-06,
+      "log_odds_chosen": 10.110807418823242,
+      "log_odds_ratio": -0.003054619999602437,
+      "logits/chosen": -1.925559401512146,
+      "logits/rejected": -1.924526572227478,
+      "logps/chosen": -0.03492622449994087,
+      "logps/rejected": -4.048843860626221,
+      "loss": 0.0415,
+      "nll_loss": 0.040362752974033356,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.001746311434544623,
+      "rewards/margins": 0.20069590210914612,
+      "rewards/rejected": -0.2024422138929367,
+      "step": 1180
+    },
+    {
+      "epoch": 2.42,
+      "grad_norm": 2.53125,
+      "learning_rate": 1.4494275891311214e-06,
+      "log_odds_chosen": 9.7469482421875,
+      "log_odds_ratio": -0.012390781193971634,
+      "logits/chosen": -1.8723634481430054,
+      "logits/rejected": -1.8694031238555908,
+      "logps/chosen": -0.045554567128419876,
+      "logps/rejected": -4.15027379989624,
+      "loss": 0.0354,
+      "nll_loss": 0.03426050767302513,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.002277728170156479,
+      "rewards/margins": 0.20523595809936523,
+      "rewards/rejected": -0.2075137197971344,
+      "step": 1190
+    },
+    {
+      "epoch": 2.44,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.4433756729740647e-06,
+      "log_odds_chosen": 10.098226547241211,
+      "log_odds_ratio": -0.0019627574365586042,
+      "logits/chosen": -1.9441982507705688,
+      "logits/rejected": -1.9426124095916748,
+      "logps/chosen": -0.02355217933654785,
+      "logps/rejected": -4.209158897399902,
+      "loss": 0.0457,
+      "nll_loss": 0.053135812282562256,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0011776090832427144,
+      "rewards/margins": 0.20928029716014862,
+      "rewards/rejected": -0.2104579210281372,
+      "step": 1200
+    },
+    {
+      "epoch": 2.46,
+      "grad_norm": 3.609375,
+      "learning_rate": 1.4373989364401727e-06,
+      "log_odds_chosen": 9.691657066345215,
+      "log_odds_ratio": -0.004629252012819052,
+      "logits/chosen": -1.9337596893310547,
+      "logits/rejected": -1.931654930114746,
+      "logps/chosen": -0.06260120123624802,
+      "logps/rejected": -4.166058540344238,
+      "loss": 0.0411,
+      "nll_loss": 0.03751087933778763,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003130060387775302,
+      "rewards/margins": 0.20517286658287048,
+      "rewards/rejected": -0.20830294489860535,
+      "step": 1210
+    },
+    {
+      "epoch": 2.48,
+      "grad_norm": 1.3984375,
+      "learning_rate": 1.4314958357846706e-06,
+      "log_odds_chosen": 9.472856521606445,
+      "log_odds_ratio": -0.00650912756100297,
+      "logits/chosen": -1.95502507686615,
+      "logits/rejected": -1.951743721961975,
+      "logps/chosen": -0.07376778870820999,
+      "logps/rejected": -4.022158145904541,
+      "loss": 0.0357,
+      "nll_loss": 0.03547949343919754,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003688389901071787,
+      "rewards/margins": 0.19741952419281006,
+      "rewards/rejected": -0.2011079341173172,
+      "step": 1220
+    },
+    {
+      "epoch": 2.5,
+      "grad_norm": 2.59375,
+      "learning_rate": 1.4256648712805027e-06,
+      "log_odds_chosen": 10.319469451904297,
+      "log_odds_ratio": -0.001875861780717969,
+      "logits/chosen": -1.9282852411270142,
+      "logits/rejected": -1.9243885278701782,
+      "logps/chosen": -0.0374315045773983,
+      "logps/rejected": -4.336482524871826,
+      "loss": 0.0412,
+      "nll_loss": 0.0424610860645771,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0018715756013989449,
+      "rewards/margins": 0.21495255827903748,
+      "rewards/rejected": -0.21682414412498474,
+      "step": 1230
+    },
+    {
+      "epoch": 2.52,
+      "grad_norm": 3.78125,
+      "learning_rate": 1.419904585617662e-06,
+      "log_odds_chosen": 10.347482681274414,
+      "log_odds_ratio": -0.0033975064288824797,
+      "logits/chosen": -1.9401060342788696,
+      "logits/rejected": -1.9385757446289062,
+      "logps/chosen": -0.04415961354970932,
+      "logps/rejected": -4.426904201507568,
+      "loss": 0.0403,
+      "nll_loss": 0.04613417014479637,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0022079809568822384,
+      "rewards/margins": 0.21913722157478333,
+      "rewards/rejected": -0.2213451862335205,
+      "step": 1240
+    },
+    {
+      "epoch": 2.54,
+      "grad_norm": 2.53125,
+      "learning_rate": 1.4142135623730952e-06,
+      "log_odds_chosen": 10.331408500671387,
+      "log_odds_ratio": -0.0018885558238252997,
+      "logits/chosen": -1.940860390663147,
+      "logits/rejected": -1.9384444952011108,
+      "logps/chosen": -0.03379340097308159,
+      "logps/rejected": -4.056138038635254,
+      "loss": 0.0393,
+      "nll_loss": 0.030756259337067604,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0016896702582016587,
+      "rewards/margins": 0.20111723244190216,
+      "rewards/rejected": -0.20280690491199493,
+      "step": 1250
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 2.109375,
+      "learning_rate": 1.4085904245475275e-06,
+      "log_odds_chosen": 10.41731071472168,
+      "log_odds_ratio": -0.012277336791157722,
+      "logits/chosen": -1.9542973041534424,
+      "logits/rejected": -1.953238844871521,
+      "logps/chosen": -0.030469322577118874,
+      "logps/rejected": -4.482726097106934,
+      "loss": 0.0407,
+      "nll_loss": 0.04968537390232086,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.001523466082289815,
+      "rewards/margins": 0.22261285781860352,
+      "rewards/rejected": -0.2241363227367401,
+      "step": 1260
+    },
+    {
+      "epoch": 2.58,
+      "grad_norm": 2.75,
+      "learning_rate": 1.4030338331657844e-06,
+      "log_odds_chosen": 10.663671493530273,
+      "log_odds_ratio": -0.005900475196540356,
+      "logits/chosen": -1.8991870880126953,
+      "logits/rejected": -1.8935034275054932,
+      "logps/chosen": -0.036128245294094086,
+      "logps/rejected": -4.496649742126465,
+      "loss": 0.036,
+      "nll_loss": 0.032302360981702805,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.00180641224142164,
+      "rewards/margins": 0.2230260670185089,
+      "rewards/rejected": -0.22483249008655548,
+      "step": 1270
+    },
+    {
+      "epoch": 2.6,
+      "grad_norm": 1.984375,
+      "learning_rate": 1.3975424859373688e-06,
+      "log_odds_chosen": 9.953948020935059,
+      "log_odds_ratio": -0.002804636722430587,
+      "logits/chosen": -1.9552701711654663,
+      "logits/rejected": -1.9517933130264282,
+      "logps/chosen": -0.04314829781651497,
+      "logps/rejected": -4.2899580001831055,
+      "loss": 0.0389,
+      "nll_loss": 0.03743810951709747,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.002157414797693491,
+      "rewards/margins": 0.21234047412872314,
+      "rewards/rejected": -0.21449792385101318,
+      "step": 1280
+    },
+    {
+      "epoch": 2.62,
+      "grad_norm": 2.046875,
+      "learning_rate": 1.3921151159742616e-06,
+      "log_odds_chosen": 9.60466194152832,
+      "log_odds_ratio": -0.021028626710176468,
+      "logits/chosen": -1.8756396770477295,
+      "logits/rejected": -1.8728761672973633,
+      "logps/chosen": -0.043241944164037704,
+      "logps/rejected": -4.110201835632324,
+      "loss": 0.0374,
+      "nll_loss": 0.03517068549990654,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0021620974875986576,
+      "rewards/margins": 0.20334799587726593,
+      "rewards/rejected": -0.20551009476184845,
+      "step": 1290
+    },
+    {
+      "epoch": 2.64,
+      "grad_norm": 2.171875,
+      "learning_rate": 1.386750490563073e-06,
+      "log_odds_chosen": 10.577958106994629,
+      "log_odds_ratio": -0.0030032056383788586,
+      "logits/chosen": -1.9832937717437744,
+      "logits/rejected": -1.9783260822296143,
+      "logps/chosen": -0.06359104067087173,
+      "logps/rejected": -4.623613357543945,
+      "loss": 0.0396,
+      "nll_loss": 0.04861373454332352,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.003179552499204874,
+      "rewards/margins": 0.22800114750862122,
+      "rewards/rejected": -0.23118066787719727,
+      "step": 1300
+    },
+    {
+      "epoch": 2.66,
+      "grad_norm": 4.1875,
+      "learning_rate": 1.3814474099888442e-06,
+      "log_odds_chosen": 10.487229347229004,
+      "log_odds_ratio": -0.0037563578225672245,
+      "logits/chosen": -1.9412161111831665,
+      "logits/rejected": -1.9390010833740234,
+      "logps/chosen": -0.04359542205929756,
+      "logps/rejected": -4.6636810302734375,
+      "loss": 0.0433,
+      "nll_loss": 0.03972161188721657,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0021797711960971355,
+      "rewards/margins": 0.2310042679309845,
+      "rewards/rejected": -0.2331840544939041,
+      "step": 1310
+    },
+    {
+      "epoch": 2.68,
+      "grad_norm": 1.796875,
+      "learning_rate": 1.376204706407951e-06,
+      "log_odds_chosen": 10.476730346679688,
+      "log_odds_ratio": -0.002192417625337839,
+      "logits/chosen": -1.9325330257415771,
+      "logits/rejected": -1.930166244506836,
+      "logps/chosen": -0.02953045442700386,
+      "logps/rejected": -4.279176712036133,
+      "loss": 0.0353,
+      "nll_loss": 0.03177348151803017,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.001476522651501,
+      "rewards/margins": 0.2124823033809662,
+      "rewards/rejected": -0.21395882964134216,
+      "step": 1320
+    },
+    {
+      "epoch": 2.7,
+      "grad_norm": 2.484375,
+      "learning_rate": 1.3710212427677044e-06,
+      "log_odds_chosen": 10.246335983276367,
+      "log_odds_ratio": -0.0470537394285202,
+      "logits/chosen": -1.9275156259536743,
+      "logits/rejected": -1.9267303943634033,
+      "logps/chosen": -0.07270321995019913,
+      "logps/rejected": -4.131697177886963,
+      "loss": 0.036,
+      "nll_loss": 0.031709469854831696,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.0036351613234728575,
+      "rewards/margins": 0.20294968783855438,
+      "rewards/rejected": -0.2065848559141159,
+      "step": 1330
+    },
+    {
+      "epoch": 2.72,
+      "grad_norm": 2.578125,
+      "learning_rate": 1.3658959117703826e-06,
+      "log_odds_chosen": 10.568267822265625,
+      "log_odds_ratio": -0.003112158738076687,
+      "logits/chosen": -1.948743462562561,
+      "logits/rejected": -1.9471248388290405,
+      "logps/chosen": -0.028313973918557167,
+      "logps/rejected": -4.5648627281188965,
+      "loss": 0.0378,
+      "nll_loss": 0.041449058800935745,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.001415698672644794,
+      "rewards/margins": 0.22682742774486542,
+      "rewards/rejected": -0.2282431572675705,
+      "step": 1340
+    },
+    {
+      "epoch": 2.74,
+      "grad_norm": 3.546875,
+      "learning_rate": 1.3608276348795436e-06,
+      "log_odds_chosen": 10.491666793823242,
+      "log_odds_ratio": -0.0048934281803667545,
+      "logits/chosen": -1.9433883428573608,
+      "logits/rejected": -1.9420783519744873,
+      "logps/chosen": -0.03949974104762077,
+      "logps/rejected": -4.376409530639648,
+      "loss": 0.0414,
+      "nll_loss": 0.03402886912226677,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.001974987331777811,
+      "rewards/margins": 0.21684548258781433,
+      "rewards/rejected": -0.2188204824924469,
+      "step": 1350
+    },
+    {
+      "epoch": 2.76,
+      "grad_norm": 2.265625,
+      "learning_rate": 1.355815361366601e-06,
+      "log_odds_chosen": 10.092164993286133,
+      "log_odds_ratio": -0.008559630252420902,
+      "logits/chosen": -1.945858359336853,
+      "logits/rejected": -1.9418405294418335,
+      "logps/chosen": -0.06214701011776924,
+      "logps/rejected": -4.4873552322387695,
+      "loss": 0.0359,
+      "nll_loss": 0.03512818366289139,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0031073507852852345,
+      "rewards/margins": 0.2212604284286499,
+      "rewards/rejected": -0.22436778247356415,
+      "step": 1360
+    },
+    {
+      "epoch": 2.78,
+      "grad_norm": 1.9296875,
+      "learning_rate": 1.350858067395748e-06,
+      "log_odds_chosen": 10.725711822509766,
+      "log_odds_ratio": -0.020754020661115646,
+      "logits/chosen": -1.9590380191802979,
+      "logits/rejected": -1.953980803489685,
+      "logps/chosen": -0.08093880116939545,
+      "logps/rejected": -4.96276330947876,
+      "loss": 0.0419,
+      "nll_loss": 0.0406344048678875,
+      "rewards/accuracies": 0.987500011920929,
+      "rewards/chosen": -0.004046940244734287,
+      "rewards/margins": 0.2440912276506424,
+      "rewards/rejected": -0.2481381893157959,
+      "step": 1370
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 3.359375,
+      "learning_rate": 1.345954755145414e-06,
+      "log_odds_chosen": 10.852327346801758,
+      "log_odds_ratio": -0.0012642950750887394,
+      "logits/chosen": -1.8603143692016602,
+      "logits/rejected": -1.85476815700531,
+      "logps/chosen": -0.03734128549695015,
+      "logps/rejected": -4.723127841949463,
+      "loss": 0.04,
+      "nll_loss": 0.025891494005918503,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0018670640420168638,
+      "rewards/margins": 0.23428933322429657,
+      "rewards/rejected": -0.2361563891172409,
+      "step": 1380
+    },
+    {
+      "epoch": 2.83,
+      "grad_norm": 2.390625,
+      "learning_rate": 1.3411044519645502e-06,
+      "log_odds_chosen": 10.230730056762695,
+      "log_odds_ratio": -0.002351417439058423,
+      "logits/chosen": -1.8683515787124634,
+      "logits/rejected": -1.86406672000885,
+      "logps/chosen": -0.044098712503910065,
+      "logps/rejected": -4.59433126449585,
+      "loss": 0.0331,
+      "nll_loss": 0.03638423979282379,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0022049355320632458,
+      "rewards/margins": 0.22751164436340332,
+      "rewards/rejected": -0.22971658408641815,
+      "step": 1390
+    },
+    {
+      "epoch": 2.85,
+      "grad_norm": 2.296875,
+      "learning_rate": 1.3363062095621222e-06,
+      "log_odds_chosen": 9.968645095825195,
+      "log_odds_ratio": -0.004948216024786234,
+      "logits/chosen": -1.9218571186065674,
+      "logits/rejected": -1.9199250936508179,
+      "logps/chosen": -0.06343747675418854,
+      "logps/rejected": -4.219951629638672,
+      "loss": 0.0353,
+      "nll_loss": 0.03614867106080055,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0031718737445771694,
+      "rewards/margins": 0.20782573521137238,
+      "rewards/rejected": -0.21099761128425598,
+      "step": 1400
+    },
+    {
+      "epoch": 2.87,
+      "grad_norm": 2.609375,
+      "learning_rate": 1.3315591032282687e-06,
+      "log_odds_chosen": 10.9727783203125,
+      "log_odds_ratio": -0.004531107842922211,
+      "logits/chosen": -1.9281492233276367,
+      "logits/rejected": -1.9244182109832764,
+      "logps/chosen": -0.03406985104084015,
+      "logps/rejected": -4.806418418884277,
+      "loss": 0.0409,
+      "nll_loss": 0.04478253796696663,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0017034925986081362,
+      "rewards/margins": 0.2386174201965332,
+      "rewards/rejected": -0.24032095074653625,
+      "step": 1410
+    },
+    {
+      "epoch": 2.89,
+      "grad_norm": 2.21875,
+      "learning_rate": 1.3268622310856882e-06,
+      "log_odds_chosen": 11.176974296569824,
+      "log_odds_ratio": -0.003099204506725073,
+      "logits/chosen": -1.9768095016479492,
+      "logits/rejected": -1.9735969305038452,
+      "logps/chosen": -0.03214428573846817,
+      "logps/rejected": -4.844766616821289,
+      "loss": 0.0368,
+      "nll_loss": 0.03034246899187565,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0016072141006588936,
+      "rewards/margins": 0.240631103515625,
+      "rewards/rejected": -0.24223831295967102,
+      "step": 1420
+    },
+    {
+      "epoch": 2.91,
+      "grad_norm": 3.953125,
+      "learning_rate": 1.3222147133698626e-06,
+      "log_odds_chosen": 9.971961975097656,
+      "log_odds_ratio": -0.009039332158863544,
+      "logits/chosen": -1.8924024105072021,
+      "logits/rejected": -1.8878147602081299,
+      "logps/chosen": -0.05458871275186539,
+      "logps/rejected": -4.532398700714111,
+      "loss": 0.041,
+      "nll_loss": 0.033775750547647476,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.002729435684159398,
+      "rewards/margins": 0.2238904982805252,
+      "rewards/rejected": -0.2266199290752411,
+      "step": 1430
+    },
+    {
+      "epoch": 2.93,
+      "grad_norm": 2.09375,
+      "learning_rate": 1.3176156917368248e-06,
+      "log_odds_chosen": 10.354188919067383,
+      "log_odds_ratio": -0.0086409542709589,
+      "logits/chosen": -1.871347427368164,
+      "logits/rejected": -1.8664233684539795,
+      "logps/chosen": -0.0295790396630764,
+      "logps/rejected": -4.4826579093933105,
+      "loss": 0.0313,
+      "nll_loss": 0.030693087726831436,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0014789520064368844,
+      "rewards/margins": 0.22265395522117615,
+      "rewards/rejected": -0.22413286566734314,
+      "step": 1440
+    },
+    {
+      "epoch": 2.95,
+      "grad_norm": 2.578125,
+      "learning_rate": 1.3130643285972255e-06,
+      "log_odds_chosen": 10.23839282989502,
+      "log_odds_ratio": -0.008670750074088573,
+      "logits/chosen": -1.9214890003204346,
+      "logits/rejected": -1.9176349639892578,
+      "logps/chosen": -0.04931662604212761,
+      "logps/rejected": -4.222093105316162,
+      "loss": 0.0354,
+      "nll_loss": 0.03125245124101639,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.002465831348672509,
+      "rewards/margins": 0.20863883197307587,
+      "rewards/rejected": -0.21110467612743378,
+      "step": 1450
+    },
+    {
+      "epoch": 2.97,
+      "grad_norm": 1.890625,
+      "learning_rate": 1.3085598064755342e-06,
+      "log_odds_chosen": 10.399072647094727,
+      "log_odds_ratio": -0.003747476963326335,
+      "logits/chosen": -1.8761295080184937,
+      "logits/rejected": -1.8740062713623047,
+      "logps/chosen": -0.048522982746362686,
+      "logps/rejected": -4.560256004333496,
+      "loss": 0.0359,
+      "nll_loss": 0.042360566556453705,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0024261490907520056,
+      "rewards/margins": 0.2255866527557373,
+      "rewards/rejected": -0.2280128002166748,
+      "step": 1460
+    },
+    {
+      "epoch": 2.99,
+      "grad_norm": 2.140625,
+      "learning_rate": 1.3041013273932528e-06,
+      "log_odds_chosen": 11.28456974029541,
+      "log_odds_ratio": -0.0021586515940725803,
+      "logits/chosen": -1.9435927867889404,
+      "logits/rejected": -1.9409191608428955,
+      "logps/chosen": -0.03155381605029106,
+      "logps/rejected": -5.088595390319824,
+      "loss": 0.0357,
+      "nll_loss": 0.028978243470191956,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.0015776908257976174,
+      "rewards/margins": 0.25285208225250244,
+      "rewards/rejected": -0.25442975759506226,
+      "step": 1470
+    },
+    {
+      "epoch": 3.0,
+      "step": 1476,
+      "total_flos": 0.0,
+      "train_loss": 0.1731818498170311,
+      "train_runtime": 10691.148,
+      "train_samples_per_second": 8.831,
+      "train_steps_per_second": 0.138
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1476,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 500,
+  "total_flos": 0.0,
+  "train_batch_size": 8,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e60dea34ea29f59d1e18b2a0572c65d481d8e1906b0ca685e3f89aa8e1c7e981
+size 5240