Training in progress, step 100000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4ed51e1ba85f76a635d5a4b2a2391f7569c86dcbec7e8462be43c05a028f2b92
+oid sha256:98759b2d7ebc1d279e01894e895488231ca886295ea5febbef75377b3c17846e
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8ff468b57d7d38029e80e932afebd1303c6c991cc99b4531d000cb90d3c844ac
+oid sha256:1cfe49edb2c5cdd32ea816756e7d42933808a9b4a91e32bcea425f1ff6d3320f
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:35f5729f689be2bbffa9964aeb321f5783ac9d47f55dce0a0ce5ba77dbc5b474
+oid sha256:23cc08fa295f1f61f14d032299f4a12f1ac03252919134b2e30f6532808f7457
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:aa6c62e484d18fd65a9b7d67344c5142e2d88165c2f53674e02273b93133c9c8
+oid sha256:18ea3abbc14ac3d713f060ff2fd846183e96ea3ca75c3d268b653d81e7259f7c
 size 1064

last-checkpoint/trainer_state.json

@@ -1,9 +1,9 @@
 {
   "best_metric": 0.5980878186968839,
   "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_cosine/checkpoint-95000",
-  "epoch": 9.370684553166305,
+  "epoch": 9.863878477017163,
   "eval_steps": 2500,
-  "global_step": 95000,
+  "global_step": 100000,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
@@ -1925,6 +1925,160 @@
       "eval_tanimoto_similarity": 0.0,
       "eval_valid": 0.9279390934844193,
       "step": 95000
+    },
+    {
+      "epoch": 9.420003945551391,
+      "grad_norm": 0.0018721942324191332,
+      "learning_rate": 2.526870501993378e-07,
+      "loss": 0.0002,
+      "step": 95500
+    },
+    {
+      "epoch": 9.469323337936476,
+      "grad_norm": 0.0017144909361377358,
+      "learning_rate": 2.0009955530982117e-07,
+      "loss": 0.0002,
+      "step": 96000
+    },
+    {
+      "epoch": 9.518642730321563,
+      "grad_norm": 0.0021374619100242853,
+      "learning_rate": 1.535318123066204e-07,
+      "loss": 0.0002,
+      "step": 96500
+    },
+    {
+      "epoch": 9.567962122706648,
+      "grad_norm": 0.005559207871556282,
+      "learning_rate": 1.1310684269457139e-07,
+      "loss": 0.0002,
+      "step": 97000
+    },
+    {
+      "epoch": 9.617281515091735,
+      "grad_norm": 0.0035916175693273544,
+      "learning_rate": 7.883464070810032e-08,
+      "loss": 0.0002,
+      "step": 97500
+    },
+    {
+      "epoch": 9.617281515091735,
+      "eval_all_ligands_equal": 0.5979992917847026,
+      "eval_e3_equal": 0.8281692634560907,
+      "eval_e3_graph_edit_distance": Infinity,
+      "eval_e3_graph_edit_distance_norm": Infinity,
+      "eval_e3_has_attachment_point(s)": 0.9868980169971672,
+      "eval_e3_heavy_atoms_difference": 0.40625,
+      "eval_e3_heavy_atoms_difference_norm": 0.008543233459256977,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9868980169971672,
+      "eval_has_all_attachment_points": 0.9848618980169972,
+      "eval_has_three_substructures": 0.9992917847025495,
+      "eval_heavy_atoms_difference": 6.117032577903683,
+      "eval_heavy_atoms_difference_norm": 0.08171100798099956,
+      "eval_linker_equal": 0.8498583569405099,
+      "eval_linker_graph_edit_distance": 2.2131728045325776e+61,
+      "eval_linker_graph_edit_distance_norm": Infinity,
+      "eval_linker_has_attachment_point(s)": 0.9977868271954674,
+      "eval_linker_heavy_atoms_difference": 0.15430240793201133,
+      "eval_linker_heavy_atoms_difference_norm": -0.0017717876495697477,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9977868271954674,
+      "eval_loss": 0.38293081521987915,
+      "eval_num_fragments": 3.0001770538243626,
+      "eval_poi_equal": 0.7955028328611898,
+      "eval_poi_graph_edit_distance": Infinity,
+      "eval_poi_graph_edit_distance_norm": Infinity,
+      "eval_poi_has_attachment_point(s)": 0.9394475920679887,
+      "eval_poi_heavy_atoms_difference": 1.805771954674221,
+      "eval_poi_heavy_atoms_difference_norm": 0.057646246942032714,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9394475920679887,
+      "eval_reassembly": 0.605435552407932,
+      "eval_reassembly_nostereo": 0.6317280453257791,
+      "eval_runtime": 2300.5871,
+      "eval_samples_per_second": 4.91,
+      "eval_steps_per_second": 0.077,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9265226628895185,
+      "step": 97500
+    },
+    {
+      "epoch": 9.66660090747682,
+      "grad_norm": 0.0018907383782789111,
+      "learning_rate": 5.0723679437578165e-08,
+      "loss": 0.0002,
+      "step": 98000
+    },
+    {
+      "epoch": 9.715920299861907,
+      "grad_norm": 0.00224716286174953,
+      "learning_rate": 2.8818634561947334e-08,
+      "loss": 0.0002,
+      "step": 98500
+    },
+    {
+      "epoch": 9.765239692246992,
+      "grad_norm": 0.0026547384914010763,
+      "learning_rate": 1.3037127690179729e-08,
+      "loss": 0.0002,
+      "step": 99000
+    },
+    {
+      "epoch": 9.814559084632076,
+      "grad_norm": 0.005669564940035343,
+      "learning_rate": 3.4331286033528754e-09,
+      "loss": 0.0002,
+      "step": 99500
+    },
+    {
+      "epoch": 9.863878477017163,
+      "grad_norm": 0.003081223228946328,
+      "learning_rate": 9.011690840776687e-12,
+      "loss": 0.0002,
+      "step": 100000
+    },
+    {
+      "epoch": 9.863878477017163,
+      "eval_all_ligands_equal": 0.5980878186968839,
+      "eval_e3_equal": 0.828257790368272,
+      "eval_e3_graph_edit_distance": Infinity,
+      "eval_e3_graph_edit_distance_norm": Infinity,
+      "eval_e3_has_attachment_point(s)": 0.9870750708215298,
+      "eval_e3_heavy_atoms_difference": 0.40421388101983,
+      "eval_e3_heavy_atoms_difference_norm": 0.008500938679118707,
+      "eval_e3_tanimoto_similarity": 0.0,
+      "eval_e3_valid": 0.9870750708215298,
+      "eval_has_all_attachment_points": 0.9846848441926346,
+      "eval_has_three_substructures": 0.9992032577903682,
+      "eval_heavy_atoms_difference": 6.106409348441926,
+      "eval_heavy_atoms_difference_norm": 0.0815233025391124,
+      "eval_linker_equal": 0.849592776203966,
+      "eval_linker_graph_edit_distance": Infinity,
+      "eval_linker_graph_edit_distance_norm": Infinity,
+      "eval_linker_has_attachment_point(s)": 0.9977868271954674,
+      "eval_linker_heavy_atoms_difference": 0.15465651558073654,
+      "eval_linker_heavy_atoms_difference_norm": -0.0017519301091180534,
+      "eval_linker_tanimoto_similarity": 0.0,
+      "eval_linker_valid": 0.9977868271954674,
+      "eval_loss": 0.38294461369514465,
+      "eval_num_fragments": 3.0000885269121813,
+      "eval_poi_equal": 0.7955028328611898,
+      "eval_poi_graph_edit_distance": Infinity,
+      "eval_poi_graph_edit_distance_norm": Infinity,
+      "eval_poi_has_attachment_point(s)": 0.9393590651558074,
+      "eval_poi_heavy_atoms_difference": 1.8055949008498584,
+      "eval_poi_heavy_atoms_difference_norm": 0.05770569057556065,
+      "eval_poi_tanimoto_similarity": 0.0,
+      "eval_poi_valid": 0.9393590651558074,
+      "eval_reassembly": 0.6055240793201133,
+      "eval_reassembly_nostereo": 0.6318165722379604,
+      "eval_runtime": 2300.3904,
+      "eval_samples_per_second": 4.91,
+      "eval_steps_per_second": 0.077,
+      "eval_tanimoto_similarity": 0.0,
+      "eval_valid": 0.9266997167138811,
+      "step": 100000
     }
   ],
   "logging_steps": 500,
@@ -1939,12 +2093,12 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": false
+        "should_training_stop": true
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.229984794717169e+17,
+  "total_flos": 8.663173632603694e+17,
   "train_batch_size": 128,
   "trial_name": null,
   "trial_params": null