Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ef49e8f550cab783de7a0ba94f48c1f7540809bf316b953689dd80be1a5dd39a
+oid sha256:b5c048fadaf20fd7a08e650cba1230d4ca450bcea0914f7ac5ee473c2e969487
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1fa9696a1cc6991b07a657bebe540a630458ccc038ac0c9a9107832507d142ed
+oid sha256:6e972b6c229380edc9a1874d6f6050ec4e8c9f61540dc012553a858ba139b946
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da000c9e378f4a1d47259777e81cf5543a01e26fac996b3222d393e65ab9bfec
+oid sha256:a3cc2cf51d792eb478c0268b254438e7c777f311f492edc1dc233e82f276b28f
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2e531f97d95afebd39a8f25f638079450f8fd8f16cc178cd6d0e1cd555754a19
+oid sha256:bae27f52aa5e4f7dfc11b827b12e1778787be379c764e4d29b5ce567fb552039
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.4987185152452497,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=22-learning_rate=7.8e-05-warmup_ratio=0.050-num_cycles=7/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 0.5648254529385771,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=22-learning_rate=7.8e-05-warmup_ratio=0.050-num_cycles=7/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.9245249668581529,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.09651552885770798,
+      "learning_rate": 6.154605555214832e-06,
+      "loss": 0.0057,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.1872074007987976,
+      "learning_rate": 7.2034075721712e-05,
+      "loss": 0.0076,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.08568382263183594,
+      "learning_rate": 3.5697508307174225e-05,
+      "loss": 0.008,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 0.0762503519654274,
+      "learning_rate": 2.777177503559546e-06,
+      "loss": 0.0037,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 0.128959521651268,
+      "learning_rate": 6.766395793785041e-05,
+      "loss": 0.006,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.5049049933716305,
+      "eval_e3_equal": 0.7996464869642068,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9888643393725144,
+      "eval_e3_heavy_atoms_difference": 0.5322138753866549,
+      "eval_e3_heavy_atoms_difference_norm": 0.011503559133904656,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9888643393725144,
+      "eval_has_all_attachment_points": 0.9821475916924437,
+      "eval_has_three_substructures": 0.9986743261157756,
+      "eval_heavy_atoms_difference": 8.533451171011931,
+      "eval_heavy_atoms_difference_norm": 0.11147858522514467,
+      "eval_linker_equal": 0.7364560318161733,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.994962439239947,
+      "eval_linker_heavy_atoms_difference": 0.4474591250552364,
+      "eval_linker_heavy_atoms_difference_norm": 0.0052732646233331305,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.994962439239947,
+      "eval_loss": 0.3063560128211975,
+      "eval_num_fragments": 2.998851082633672,
+      "eval_poi_equal": 0.7368095448519664,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9110030932390631,
+      "eval_poi_heavy_atoms_difference": 2.787185152452497,
+      "eval_poi_heavy_atoms_difference_norm": 0.0886750621661839,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9110030932390631,
+      "eval_reassembly": 0.5122403888643394,
+      "eval_reassembly_nostereo": 0.5371630578877596,
+      "eval_runtime": 1510.5044,
+      "eval_samples_per_second": 7.491,
+      "eval_steps_per_second": 0.117,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.8995139195757843,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.09418843686580658,
+      "learning_rate": 2.8438833453583697e-05,
+      "loss": 0.0047,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.0902002677321434,
+      "learning_rate": 7.006249789397529e-07,
+      "loss": 0.0026,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.12980258464813232,
+      "learning_rate": 6.224953837527164e-05,
+      "loss": 0.0066,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 0.048701103776693344,
+      "learning_rate": 2.155355502020242e-05,
+      "loss": 0.0044,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.06892744451761246,
+      "learning_rate": 0.0,
+      "loss": 0.002,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.5648254529385771,
+      "eval_e3_equal": 0.811842686699072,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9891294741493593,
+      "eval_e3_heavy_atoms_difference": 0.5316836058329651,
+      "eval_e3_heavy_atoms_difference_norm": 0.011111080767439089,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9891294741493593,
+      "eval_has_all_attachment_points": 0.9803800265134777,
+      "eval_has_three_substructures": 0.9994697304463102,
+      "eval_heavy_atoms_difference": 6.5374281926646045,
+      "eval_heavy_atoms_difference_norm": 0.08639325931869864,
+      "eval_linker_equal": 0.8051259390190013,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9975254087494476,
+      "eval_linker_heavy_atoms_difference": 0.28652231551038443,
+      "eval_linker_heavy_atoms_difference_norm": 0.001050129570856685,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9975254087494476,
+      "eval_loss": 0.29083964228630066,
+      "eval_num_fragments": 2.99946973044631,
+      "eval_poi_equal": 0.7801148917366328,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9341581970835174,
+      "eval_poi_heavy_atoms_difference": 1.9076447193990278,
+      "eval_poi_heavy_atoms_difference_norm": 0.06058139871085425,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9341581970835174,
+      "eval_reassembly": 0.5731330092797172,
+      "eval_reassembly_nostereo": 0.6010605391073796,
+      "eval_runtime": 1507.1636,
+      "eval_samples_per_second": 7.507,
+      "eval_steps_per_second": 0.117,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9225806451612903,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.33139477262336e+16,
+  "total_flos": 8.663036348367667e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=22-learning_rate=7.8e-05-warmup_ratio=0.050-num_cycles=7",
   "trial_params": {