Training in progress, step 10000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9e4ccbcc06819732165d5bccf3cf9d54778ffdba77df1991aea0d7bbe537687
+oid sha256:8f15bc6899e10e2875d0e58ef068fabe8068ed092ffaaccf564e13f45aa5e223
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:c9c5de92db841339669f7b639a74cb1b95e8b8118cd16a2ad903b681fb269019
+oid sha256:1fb14749e522ed4c74afc370e250f0ceafeda5e9584b713160889a43634e3c31
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d01d7d352c6fbbcdc2b84624b621978f74d278764a3d186758de97a27bf5b935
+oid sha256:fdd21bd9c09685281fbf6c62ade7d74e50289f375c871724c2edc8c81e406942
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4bbc34c4bcb0bc7d443ac2b03110a018dc339475d93326d24ebc2f6aa374c034
+oid sha256:397c886399d45556b57f80d83f9cc295715699f48ae5732744568b7ce4e12ddf
 size 1192

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
-  "best_metric": 8.837825894829872e-05,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=12-learning_rate=2.1e-06-warmup_ratio=0.030-min_lr=0.000-factor=0.760/checkpoint-5000",
-  "epoch": 0.49319392385085814,
+  "best_metric": 0.0007070260715863897,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=12-learning_rate=2.1e-06-warmup_ratio=0.030-min_lr=0.000-factor=0.760/checkpoint-10000",
+  "epoch": 0.9863878477017163,
   "eval_steps": 2500,
-  "global_step": 5000,
+  "global_step": 10000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -119,6 +119,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.5147149801148917,
       "step": 5000
+    },
+    {
+      "epoch": 0.542513316235944,
+      "grad_norm": 0.8510923981666565,
+      "learning_rate": 2.079905465346972e-06,
+      "loss": 0.3895,
+      "step": 5500
+    },
+    {
+      "epoch": 0.5918327086210298,
+      "grad_norm": 0.8566566705703735,
+      "learning_rate": 2.079905465346972e-06,
+      "loss": 0.3578,
+      "step": 6000
+    },
+    {
+      "epoch": 0.6411521010061156,
+      "grad_norm": 0.7617588639259338,
+      "learning_rate": 2.079905465346972e-06,
+      "loss": 0.3306,
+      "step": 6500
+    },
+    {
+      "epoch": 0.6904714933912014,
+      "grad_norm": 1.0278968811035156,
+      "learning_rate": 2.079905465346972e-06,
+      "loss": 0.3068,
+      "step": 7000
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "grad_norm": 0.8128023147583008,
+      "learning_rate": 2.079905465346972e-06,
+      "loss": 0.2855,
+      "step": 7500
+    },
+    {
+      "epoch": 0.7397908857762873,
+      "eval_all_ligands_equal": 0.00017675651789659743,
+      "eval_e3_equal": 0.14732655766681396,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9926646045072912,
+      "eval_e3_heavy_atoms_difference": -1.3951391957578436,
+      "eval_e3_heavy_atoms_difference_norm": -0.08085056605381094,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9926646045072912,
+      "eval_has_all_attachment_points": 0.9148033583738401,
+      "eval_has_three_substructures": 0.9933716305788776,
+      "eval_heavy_atoms_difference": 30.79673000441891,
+      "eval_heavy_atoms_difference_norm": 0.39236787791885963,
+      "eval_linker_equal": 0.003977021652673442,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9880689350419797,
+      "eval_linker_heavy_atoms_difference": 5.820503756076005,
+      "eval_linker_heavy_atoms_difference_norm": 0.24138745220741845,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9880689350419797,
+      "eval_loss": 0.42061758041381836,
+      "eval_num_fragments": 2.99584622182943,
+      "eval_poi_equal": 0.09288555015466195,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.6764471939902784,
+      "eval_poi_heavy_atoms_difference": 12.387715422006186,
+      "eval_poi_heavy_atoms_difference_norm": 0.35498857497941394,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.6764471939902784,
+      "eval_reassembly": 0.0002651347768448962,
+      "eval_reassembly_nostereo": 0.00035351303579319487,
+      "eval_runtime": 1510.8447,
+      "eval_samples_per_second": 7.489,
+      "eval_steps_per_second": 0.117,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.6732655766681397,
+      "step": 7500
+    },
+    {
+      "epoch": 0.789110278161373,
+      "grad_norm": 0.9169577956199646,
+      "learning_rate": 2.079905465346972e-06,
+      "loss": 0.268,
+      "step": 8000
+    },
+    {
+      "epoch": 0.8384296705464589,
+      "grad_norm": 0.8658037781715393,
+      "learning_rate": 2.079905465346972e-06,
+      "loss": 0.2512,
+      "step": 8500
+    },
+    {
+      "epoch": 0.8877490629315447,
+      "grad_norm": 0.8686316609382629,
+      "learning_rate": 2.079905465346972e-06,
+      "loss": 0.2369,
+      "step": 9000
+    },
+    {
+      "epoch": 0.9370684553166305,
+      "grad_norm": 1.007585883140564,
+      "learning_rate": 2.079905465346972e-06,
+      "loss": 0.2234,
+      "step": 9500
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "grad_norm": 0.6619876623153687,
+      "learning_rate": 2.079905465346972e-06,
+      "loss": 0.2112,
+      "step": 10000
+    },
+    {
+      "epoch": 0.9863878477017163,
+      "eval_all_ligands_equal": 0.0007070260715863897,
+      "eval_e3_equal": 0.2,
+      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
+      "eval_e3_graph_edit_distance_norm": 1.0,
+      "eval_e3_has_attachment_point(s)": 0.9937251436146708,
+      "eval_e3_heavy_atoms_difference": -1.2498453380468404,
+      "eval_e3_heavy_atoms_difference_norm": -0.07127699571538457,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9937251436146708,
+      "eval_has_all_attachment_points": 0.9333627927529827,
+      "eval_has_three_substructures": 0.994078656650464,
+      "eval_heavy_atoms_difference": 24.293680954485197,
+      "eval_heavy_atoms_difference_norm": 0.31102480232191276,
+      "eval_linker_equal": 0.006981882456915599,
+      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
+      "eval_linker_graph_edit_distance_norm": 1.0,
+      "eval_linker_has_attachment_point(s)": 0.9891294741493593,
+      "eval_linker_heavy_atoms_difference": 5.449580203269996,
+      "eval_linker_heavy_atoms_difference_norm": 0.23420443958910875,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9891294741493593,
+      "eval_loss": 0.3799628019332886,
+      "eval_num_fragments": 2.99584622182943,
+      "eval_poi_equal": 0.18877596111356607,
+      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
+      "eval_poi_graph_edit_distance_norm": 1.0,
+      "eval_poi_has_attachment_point(s)": 0.7513919575784357,
+      "eval_poi_heavy_atoms_difference": 9.262218294299602,
+      "eval_poi_heavy_atoms_difference_norm": 0.265899326545649,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.7513919575784357,
+      "eval_reassembly": 0.0007070260715863897,
+      "eval_reassembly_nostereo": 0.0009721608484312859,
+      "eval_runtime": 1478.0556,
+      "eval_samples_per_second": 7.655,
+      "eval_steps_per_second": 0.12,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.7479452054794521,
+      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -133,12 +287,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 4.364959997072179e+16,
+  "total_flos": 8.729673191023411e+16,
   "train_batch_size": 128,
   "trial_name": "trial-number=12-learning_rate=2.1e-06-warmup_ratio=0.030-min_lr=0.000-factor=0.760",
   "trial_params": {