Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:48b0e0e392de30e27c1678fa90a3714f2ed69176dde100c22e0602c0120aef59
+oid sha256:bcdb3969386fa38fd6adc71ea7e9f7b4b777da6d435448e1643c0757a5cecbcb
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:518e736af647fcc70835e741267a15d776ec7962141ca9b539a19357ff03006f
+oid sha256:6279370528b3f60c4f30ba71f95a93e2c8f5bc7e7f6a818c6afc8689f76340f1
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7512f3014b8688aecadf72081c095e15a11bfd4f0c6939adc2147da05c208a7f
+oid sha256:ac938fc6b38a53610f05ced08e522126f1a61c094b4aedb5a6dc9ff5ced12fc0
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8a4a11b800eb83e700cf44a26c7346fc77b30b99dcb2df7384323250a5ed6a5e
+oid sha256:ff4b36dddeb8a6b98fa508682c8e66b736535bad766232baf8629362d1598122
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 0.2691117984975696,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=16-learning_rate=2.7e-05-warmup_ratio=0.020-num_cycles=4/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 0.49403446752098984,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine_restarts-opt25/trial-number=16-learning_rate=2.7e-05-warmup_ratio=0.020-num_cycles=4/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.9043747238179408,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.28343096375465393,
+      "learning_rate": 2.451696899615336e-05,
+      "loss": 0.0365,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.22572530806064606,
+      "learning_rate": 1.806743983536228e-05,
+      "loss": 0.0269,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.20301209390163422,
+      "learning_rate": 9.767171357425924e-06,
+      "loss": 0.0204,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 0.1848202645778656,
+      "learning_rate": 2.8487818486075695e-06,
+      "loss": 0.0173,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 0.20325815677642822,
+      "learning_rate": 6.703723611243483e-09,
+      "loss": 0.0159,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.4345558992487848,
+      "eval_e3_equal": 0.7791427308882015,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.993813521873619,
+      "eval_e3_heavy_atoms_difference": 0.18974812196199736,
+      "eval_e3_heavy_atoms_difference_norm": -0.002278847997197217,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.993813521873619,
+      "eval_has_all_attachment_points": 0.9840919133893062,
+      "eval_has_three_substructures": 0.9990278391515687,
+      "eval_heavy_atoms_difference": 7.406186478126381,
+      "eval_heavy_atoms_difference_norm": 0.09853121268513322,
+      "eval_linker_equal": 0.6171453822359699,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9968183826778613,
+      "eval_linker_heavy_atoms_difference": 0.5403446752098984,
+      "eval_linker_heavy_atoms_difference_norm": 0.00343612583628965,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9968183826778613,
+      "eval_loss": 0.2865717113018036,
+      "eval_num_fragments": 2.999734865223155,
+      "eval_poi_equal": 0.7204595669465311,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.9159522757401679,
+      "eval_poi_heavy_atoms_difference": 2.4522315510384445,
+      "eval_poi_heavy_atoms_difference_norm": 0.07822232883011575,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9159522757401679,
+      "eval_reassembly": 0.4412726469288555,
+      "eval_reassembly_nostereo": 0.46531153336279274,
+      "eval_runtime": 1494.4607,
+      "eval_samples_per_second": 7.571,
+      "eval_steps_per_second": 0.118,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.909147149801149,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.2400677502155304,
+      "learning_rate": 2.428292541395795e-05,
+      "loss": 0.0189,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.1835109442472458,
+      "learning_rate": 1.7670398743193993e-05,
+      "loss": 0.0142,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.21109040081501007,
+      "learning_rate": 9.361764174539816e-06,
+      "loss": 0.011,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 0.15360859036445618,
+      "learning_rate": 2.5928982476405618e-06,
+      "loss": 0.0095,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.17562519013881683,
+      "learning_rate": 0.0,
+      "loss": 0.0087,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.49403446752098984,
+      "eval_e3_equal": 0.7933716305788776,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9946973044631021,
+      "eval_e3_heavy_atoms_difference": 0.1994697304463102,
+      "eval_e3_heavy_atoms_difference_norm": -0.0015405930907161603,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9946973044631021,
+      "eval_has_all_attachment_points": 0.9829429960229783,
+      "eval_has_three_substructures": 0.9992929739284137,
+      "eval_heavy_atoms_difference": 7.228988068935042,
+      "eval_heavy_atoms_difference_norm": 0.09704291267150084,
+      "eval_linker_equal": 0.6980114891736633,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9960229783473266,
+      "eval_linker_heavy_atoms_difference": 0.3879805567830314,
+      "eval_linker_heavy_atoms_difference_norm": -0.0004028315220320436,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9960229783473266,
+      "eval_loss": 0.29365620017051697,
+      "eval_num_fragments": 3.0001767565178965,
+      "eval_poi_equal": 0.7405214317277949,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.916570923552806,
+      "eval_poi_heavy_atoms_difference": 2.436411842686699,
+      "eval_poi_heavy_atoms_difference_norm": 0.08075788602113727,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.916570923552806,
+      "eval_reassembly": 0.5017233760494918,
+      "eval_reassembly_nostereo": 0.5281484754750332,
+      "eval_runtime": 1493.4927,
+      "eval_samples_per_second": 7.576,
+      "eval_steps_per_second": 0.119,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.910384445426425,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.33139477262336e+16,
+  "total_flos": 8.663036348367667e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=16-learning_rate=2.7e-05-warmup_ratio=0.020-num_cycles=4",
   "trial_params": {