Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:332cd84d15ef2fb315780519f0d582e3337e9047ea7b44dc5428de90d43e58e0
+oid sha256:cf78271e0f4e5c70ae55d947ae2a7a8831d7ef512d5ebd77f495c2b5fe701058
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9909caa6f6726f14b143a92e634bdb0ea96f4213205030190cad08d20877bbcc
+oid sha256:0d726202e9e58d549edc82d9a43cc8c8012d365e18f9f6c7228124a710750030
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f05568c9474927fa3f00b14190be9bec9b14406698bef27ced0d260778a147e1
+oid sha256:e0021e993c182c62c5e9048adf28e156de0b709098b23a1b8c3aaf8b1bda5885
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:604465ad0e65b5f81a0864bb03d101226966a1aafde4cf2fccf8bf9b80b3cb10
+oid sha256:a15d996c067d87981f4a2bdb42e38d6b9228437c7ebba92180e20a0bb4edd780
 size 1192

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.0,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=22-learning_rate=6.6e-04-warmup_ratio=0.030-min_lr=0.000-factor=0.780/checkpoint-5000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.5213433495360141,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=23-learning_rate=9.9e-05-warmup_ratio=0.070-min_lr=0.000-factor=0.120/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.648746907711029,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5973,
+      "grad_norm": 0.6064334511756897,
+      "learning_rate": 9.886132722858021e-05,
+      "loss": 0.7751,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.6236242651939392,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5692,
+      "grad_norm": 0.31307706236839294,
+      "learning_rate": 9.886132722858021e-05,
+      "loss": 0.1148,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.8311471939086914,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5695,
+      "grad_norm": 0.19195418059825897,
+      "learning_rate": 9.886132722858021e-05,
+      "loss": 0.0417,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.8946245908737183,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5652,
+      "grad_norm": 0.19379402697086334,
+      "learning_rate": 9.886132722858021e-05,
+      "loss": 0.0241,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.8499246835708618,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5648,
+      "grad_norm": 0.13165819644927979,
+      "learning_rate": 9.886132722858021e-05,
+      "loss": 0.0166,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.8524408936500549,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5627,
+      "grad_norm": 0.1423509120941162,
+      "learning_rate": 9.886132722858021e-05,
+      "loss": 0.0129,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.8769518136978149,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5602,
+      "grad_norm": 0.14043006300926208,
+      "learning_rate": 9.886132722858021e-05,
+      "loss": 0.0102,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.9180594086647034,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5602,
+      "grad_norm": 0.12152795493602753,
+      "learning_rate": 9.886132722858021e-05,
+      "loss": 0.0094,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.9766675233840942,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5586,
+      "grad_norm": 0.10019189119338989,
+      "learning_rate": 9.886132722858021e-05,
+      "loss": 0.009,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.9165359735488892,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5581,
+      "grad_norm": 0.09819556027650833,
+      "learning_rate": 9.886132722858021e-05,
+      "loss": 0.0069,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.0,
+      "eval_all_ligands_equal": 0.5213433495360141,
+      "eval_e3_equal": 0.8005302695536898,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.0,
-      "eval_e3_heavy_atoms_difference": 28.086433937251435,
-      "eval_e3_heavy_atoms_difference_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9891294741493593,
+      "eval_e3_heavy_atoms_difference": 0.4190897039328325,
+      "eval_e3_heavy_atoms_difference_norm": 0.008621549797286895,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.0,
-      "eval_has_all_attachment_points": 0.0,
-      "eval_has_three_substructures": 0.0,
-      "eval_heavy_atoms_difference": 75.67370746796288,
-      "eval_heavy_atoms_difference_norm": 1.0,
-      "eval_linker_equal": 0.0,
+      "eval_e3_valid": 0.9891294741493593,
+      "eval_has_all_attachment_points": 0.9900132567388422,
+      "eval_has_three_substructures": 0.9992929739284137,
+      "eval_heavy_atoms_difference": 4.418559434379143,
+      "eval_heavy_atoms_difference_norm": 0.05692426841147494,
+      "eval_linker_equal": 0.7637649138311975,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.0,
-      "eval_linker_heavy_atoms_difference": 16.430402121078213,
-      "eval_linker_heavy_atoms_difference_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9954043305346885,
+      "eval_linker_heavy_atoms_difference": 0.4836058329650906,
+      "eval_linker_heavy_atoms_difference_norm": 0.006199250631660274,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.0,
-      "eval_loss": 3.580756664276123,
-      "eval_num_fragments": 1.0018559434379142,
-      "eval_poi_equal": 0.0,
+      "eval_linker_valid": 0.9954043305346885,
+      "eval_loss": 0.31019213795661926,
+      "eval_num_fragments": 2.999646486964207,
+      "eval_poi_equal": 0.755899248784799,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.0,
-      "eval_poi_heavy_atoms_difference": 31.15687140963323,
-      "eval_poi_heavy_atoms_difference_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9647370746796288,
+      "eval_poi_heavy_atoms_difference": 1.0888201502430401,
+      "eval_poi_heavy_atoms_difference_norm": 0.026170453342413177,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 5695.4606,
-      "eval_samples_per_second": 1.987,
-      "eval_steps_per_second": 0.031,
+      "eval_poi_valid": 0.9647370746796288,
+      "eval_reassembly": 0.5285019885108263,
+      "eval_reassembly_nostereo": 0.5531595227574017,
+      "eval_runtime": 1519.1579,
+      "eval_samples_per_second": 7.448,
+      "eval_steps_per_second": 0.117,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.0,
+      "eval_valid": 0.9515687140963323,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.9390079975128174,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5573,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.37822020053863525,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5582,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.9856277108192444,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5613,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.9074209928512573,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5565,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.9249660968780518,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5559,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.0,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.0,
-      "eval_e3_heavy_atoms_difference": 28.086433937251435,
-      "eval_e3_heavy_atoms_difference_norm": 1.0,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.0,
-      "eval_has_all_attachment_points": 0.0,
-      "eval_has_three_substructures": 0.0,
-      "eval_heavy_atoms_difference": 75.67370746796288,
-      "eval_heavy_atoms_difference_norm": 1.0,
-      "eval_linker_equal": 0.0,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.0,
-      "eval_linker_heavy_atoms_difference": 16.430402121078213,
-      "eval_linker_heavy_atoms_difference_norm": 1.0,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.0,
-      "eval_loss": 3.5794432163238525,
-      "eval_num_fragments": 1.0028281042863456,
-      "eval_poi_equal": 0.0,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.0,
-      "eval_poi_heavy_atoms_difference": 31.15687140963323,
-      "eval_poi_heavy_atoms_difference_norm": 1.0,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 5682.0008,
-      "eval_samples_per_second": 1.991,
-      "eval_steps_per_second": 0.031,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.0,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.8662272691726685,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5565,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.9421558976173401,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5563,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.826027512550354,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5562,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.883988618850708,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5554,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.8590209484100342,
-      "learning_rate": 0.0006583930582350292,
-      "loss": 3.5568,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.0,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.0,
-      "eval_e3_heavy_atoms_difference": 28.086433937251435,
-      "eval_e3_heavy_atoms_difference_norm": 1.0,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.0,
-      "eval_has_all_attachment_points": 0.0,
-      "eval_has_three_substructures": 0.0,
-      "eval_heavy_atoms_difference": 75.67370746796288,
-      "eval_heavy_atoms_difference_norm": 1.0,
-      "eval_linker_equal": 0.0,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.0,
-      "eval_linker_heavy_atoms_difference": 16.430402121078213,
-      "eval_linker_heavy_atoms_difference_norm": 1.0,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.0,
-      "eval_loss": 3.5780739784240723,
-      "eval_num_fragments": 1.0027397260273974,
-      "eval_poi_equal": 0.0,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.0,
-      "eval_poi_heavy_atoms_difference": 31.15687140963323,
-      "eval_poi_heavy_atoms_difference_norm": 1.0,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 5630.8868,
-      "eval_samples_per_second": 2.009,
-      "eval_steps_per_second": 0.031,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.0,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,18 +133,18 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=22-learning_rate=6.6e-04-warmup_ratio=0.030-min_lr=0.000-factor=0.780",
+  "trial_name": "trial-number=23-learning_rate=9.9e-05-warmup_ratio=0.070-min_lr=0.000-factor=0.120",
   "trial_params": {
-    "factor": 0.78,
-    "learning_rate": 0.0006583930582350292,
-    "min_lr": 1.4021444230605124e-10,
-    "warmup_ratio": 0.03
+    "factor": 0.12000000000000001,
+    "learning_rate": 9.886132722858021e-05,
+    "min_lr": 4.9933665355601805e-12,
+    "warmup_ratio": 0.06999999999999999
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e63068083974a265ad4a5244ae7a0a24c243685f6bdce7c9fe6a689b2b880d6f
+oid sha256:029e9519837cd9c702131829d4b707092d76dd219fffcb91e93c74c3cf895f50
 size 7544