Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ae4e0003edab8373b954265f3246fc6385e79937384366df5196b5468875793c
+oid sha256:0b242afdf88c260c9d2cddded3c1ee0887525af465dd5dcdcfdc2b84a28b6253
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a1acbbea4f5ccb9061ef6cc362f1b48917a0c85a5b28a17aa94c35d2ec1a7965
+oid sha256:80579e5bdf8b5e88cdb1dbc26c83a24925f3127ec6e0a4becdd9f1d715beb94a
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f05568c9474927fa3f00b14190be9bec9b14406698bef27ced0d260778a147e1
+oid sha256:d96f4f938570a561254bd4d8ecbd121b640c60ca00925f29b168fa68795030c8
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ef5a20ed5c83f6f7177dd81865efca8b994f3cc4ebd977db32652679c137af79
+oid sha256:8b5fc9d7aeafbcfc85ad28e800169b9873e63bee435ef84a54bca5548b7a7f37
 size 1192

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.0,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=14-learning_rate=2.4e-04-warmup_ratio=0.020-min_lr=0.000-factor=0.190/checkpoint-5000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.5035793194874061,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=15-learning_rate=4.9e-05-warmup_ratio=0.060-min_lr=0.000-factor=0.880/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 2.8864777088165283,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.4438,
+      "grad_norm": 0.6872519850730896,
+      "learning_rate": 4.908939904737889e-05,
+      "loss": 0.9674,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 3.0326426029205322,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.3548,
+      "grad_norm": 0.4604284465312958,
+      "learning_rate": 4.908939904737889e-05,
+      "loss": 0.1862,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 4.5457024574279785,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.3499,
+      "grad_norm": 0.28095778822898865,
+      "learning_rate": 4.908939904737889e-05,
+      "loss": 0.079,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 1.3327759504318237,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.3274,
+      "grad_norm": 0.25549444556236267,
+      "learning_rate": 4.908939904737889e-05,
+      "loss": 0.045,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 3.498711347579956,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.3177,
+      "grad_norm": 0.18728004395961761,
+      "learning_rate": 4.908939904737889e-05,
+      "loss": 0.0298,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 2.9555630683898926,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.3221,
+      "grad_norm": 0.1978629231452942,
+      "learning_rate": 4.908939904737889e-05,
+      "loss": 0.0218,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.8028567433357239,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.382,
+      "grad_norm": 0.17600569128990173,
+      "learning_rate": 4.908939904737889e-05,
+      "loss": 0.0169,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 2.8789641857147217,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.5139,
+      "grad_norm": 0.17753109335899353,
+      "learning_rate": 4.908939904737889e-05,
+      "loss": 0.014,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 1.7469948530197144,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.4395,
+      "grad_norm": 0.1801772564649582,
+      "learning_rate": 4.908939904737889e-05,
+      "loss": 0.0119,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 2.0340139865875244,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.4203,
+      "grad_norm": 0.1526259332895279,
+      "learning_rate": 4.908939904737889e-05,
+      "loss": 0.01,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.0,
+      "eval_all_ligands_equal": 0.5035793194874061,
+      "eval_e3_equal": 0.796730004418913,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.0,
-      "eval_e3_heavy_atoms_difference": 28.086433937251435,
-      "eval_e3_heavy_atoms_difference_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9876270437472382,
+      "eval_e3_heavy_atoms_difference": 0.49986743261157757,
+      "eval_e3_heavy_atoms_difference_norm": 0.009170609294278426,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.0,
-      "eval_has_all_attachment_points": 0.0,
-      "eval_has_three_substructures": 0.0,
-      "eval_heavy_atoms_difference": 75.67370746796288,
-      "eval_heavy_atoms_difference_norm": 1.0,
-      "eval_linker_equal": 0.0,
+      "eval_e3_valid": 0.9876270437472382,
+      "eval_has_all_attachment_points": 0.9775519222271322,
+      "eval_has_three_substructures": 0.999116217410517,
+      "eval_heavy_atoms_difference": 5.628722934158197,
+      "eval_heavy_atoms_difference_norm": 0.07344507398201826,
+      "eval_linker_equal": 0.7257622624834291,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.0,
-      "eval_linker_heavy_atoms_difference": 16.430402121078213,
-      "eval_linker_heavy_atoms_difference_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9968183826778613,
+      "eval_linker_heavy_atoms_difference": 0.6482545293857711,
+      "eval_linker_heavy_atoms_difference_norm": 0.022853714893161348,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.0,
-      "eval_loss": 8.57220458984375,
-      "eval_num_fragments": 1.0,
-      "eval_poi_equal": 0.0,
+      "eval_linker_valid": 0.9968183826778613,
+      "eval_loss": 0.2956240773200989,
+      "eval_num_fragments": 2.9992929739284135,
+      "eval_poi_equal": 0.7451171011931065,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.0,
-      "eval_poi_heavy_atoms_difference": 31.15687140963323,
-      "eval_poi_heavy_atoms_difference_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9543968183826779,
+      "eval_poi_heavy_atoms_difference": 1.4922669023420239,
+      "eval_poi_heavy_atoms_difference_norm": 0.04389473382263694,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 5677.3518,
-      "eval_samples_per_second": 1.993,
-      "eval_steps_per_second": 0.031,
+      "eval_poi_valid": 0.9543968183826779,
+      "eval_reassembly": 0.5103844454264251,
+      "eval_reassembly_nostereo": 0.5380468404772426,
+      "eval_runtime": 1503.1796,
+      "eval_samples_per_second": 7.527,
+      "eval_steps_per_second": 0.118,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.0,
+      "eval_valid": 0.9413168360583296,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 3.6723673343658447,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.4046,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 3.152406692504883,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.3936,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 5.665538311004639,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.3834,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 7.752516269683838,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.3789,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 23.502029418945312,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.3674,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.0,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.0,
-      "eval_e3_heavy_atoms_difference": 28.086433937251435,
-      "eval_e3_heavy_atoms_difference_norm": 1.0,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.0,
-      "eval_has_all_attachment_points": 0.0,
-      "eval_has_three_substructures": 0.0,
-      "eval_heavy_atoms_difference": 75.67370746796288,
-      "eval_heavy_atoms_difference_norm": 1.0,
-      "eval_linker_equal": 0.0,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.0,
-      "eval_linker_heavy_atoms_difference": 16.430402121078213,
-      "eval_linker_heavy_atoms_difference_norm": 1.0,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.0,
-      "eval_loss": 9.433979988098145,
-      "eval_num_fragments": 1.0,
-      "eval_poi_equal": 0.0,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.0,
-      "eval_poi_heavy_atoms_difference": 31.15687140963323,
-      "eval_poi_heavy_atoms_difference_norm": 1.0,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 5663.69,
-      "eval_samples_per_second": 1.998,
-      "eval_steps_per_second": 0.031,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.0,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 153.28866577148438,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.3719,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 3373.807861328125,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.3659,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 10.40761661529541,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.3619,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 65.70999908447266,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.3661,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.7575370073318481,
-      "learning_rate": 0.000238890409000628,
-      "loss": 3.4683,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.0,
-      "eval_e3_equal": 0.0,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.0,
-      "eval_e3_heavy_atoms_difference": 28.086433937251435,
-      "eval_e3_heavy_atoms_difference_norm": 1.0,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.0,
-      "eval_has_all_attachment_points": 0.0,
-      "eval_has_three_substructures": 0.0,
-      "eval_heavy_atoms_difference": 75.67370746796288,
-      "eval_heavy_atoms_difference_norm": 1.0,
-      "eval_linker_equal": 0.0,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.0,
-      "eval_linker_heavy_atoms_difference": 16.430402121078213,
-      "eval_linker_heavy_atoms_difference_norm": 1.0,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.0,
-      "eval_loss": 3.69158935546875,
-      "eval_num_fragments": 1.012372956252762,
-      "eval_poi_equal": 0.0,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.0,
-      "eval_poi_heavy_atoms_difference": 31.15687140963323,
-      "eval_poi_heavy_atoms_difference_norm": 1.0,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.0,
-      "eval_reassembly": 0.0,
-      "eval_reassembly_nostereo": 0.0,
-      "eval_runtime": 5626.3853,
-      "eval_samples_per_second": 2.011,
-      "eval_steps_per_second": 0.031,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.0,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,18 +133,18 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=14-learning_rate=2.4e-04-warmup_ratio=0.020-min_lr=0.000-factor=0.190",
+  "trial_name": "trial-number=15-learning_rate=4.9e-05-warmup_ratio=0.060-min_lr=0.000-factor=0.880",
   "trial_params": {
-    "factor": 0.19,
-    "learning_rate": 0.000238890409000628,
-    "min_lr": 1.1421601081613174e-11,
-    "warmup_ratio": 0.02
+    "factor": 0.88,
+    "learning_rate": 4.908939904737889e-05,
+    "min_lr": 3.30064578523678e-10,
+    "warmup_ratio": 0.060000000000000005
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8df19fb82993d53ffe89dfe2d59d0ee2b1ce1945d531709caa9b02e67a873074
+oid sha256:af5ecd76de05949f5bdddde52794cf5122b2bddbf2fdc7693805ee08507a7425
 size 7544