Training in progress, step 5000, checkpoint

last-checkpoint/model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0e411438c5addde45ea50e8a57c3e84c79f9588c1db1472d2cb805e009e02f83
+oid sha256:b66c7275f80108b914fa2d9ded74aab2e8e26ed98cefe3555195732ab6c3fa4c
 size 409608164

last-checkpoint/optimizer.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba40fcd259a07900f89ab37e14053e448c498293a17a232f04f00cf83e2b5bd5
+oid sha256:2e45132fc0710746ad8c9d408d8d8116f774e1274ae88a34c7b5cad8193e23d5
 size 814647162

last-checkpoint/rng_state.pth

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:251a67f4f3a3a7b943623fc65d1607ac257a3096d6b2b4d5becdd08574b7d603
+oid sha256:f6732c671f36151cdd05b97be18c12b2988cffb1f309a2cb00d544de58eb0ce2
 size 14244

last-checkpoint/scheduler.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:30e91d7e08c8765f41001a17b8cefbb71b04c6031c33176c504e767dbd051f92
+oid sha256:71ee91cdf0c467c8e19d5529367174afb8e82cb29c7b4ac8c39d892b258405ec
 size 1192

last-checkpoint/trainer_state.json

@@ -1,278 +1,124 @@
 {
-  "best_metric": 0.5585505965532479,
-  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=10-learning_rate=7.6e-05-warmup_ratio=0.040-min_lr=0.000-factor=0.940/checkpoint-10000",
-  "epoch": 0.9863878477017163,
+  "best_metric": 0.0,
+  "best_model_checkpoint": "/mimer/NOBACKUP/groups/naiss2023-6-290/stefano/models//PROTAC-Splitter-EncoderDecoder-lr_reduce-opt25-rand-smiles/trial-number=11-learning_rate=8.7e-04-warmup_ratio=0.080-min_lr=0.000-factor=0.280/checkpoint-5000",
+  "epoch": 0.49319392385085814,
   "eval_steps": 2500,
-  "global_step": 10000,
+  "global_step": 5000,
   "is_hyper_param_search": true,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
       "epoch": 0.049319392385085814,
-      "grad_norm": 0.7516372799873352,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.8486,
+      "grad_norm": 0.632607638835907,
+      "learning_rate": 0.0008697519738924375,
+      "loss": 3.5948,
       "step": 500
     },
     {
       "epoch": 0.09863878477017163,
-      "grad_norm": 0.3815319538116455,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.1503,
+      "grad_norm": 0.5518113970756531,
+      "learning_rate": 0.0008697519738924375,
+      "loss": 3.568,
       "step": 1000
     },
     {
       "epoch": 0.14795817715525744,
-      "grad_norm": 0.22006957232952118,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.0563,
+      "grad_norm": 0.8580166697502136,
+      "learning_rate": 0.0008697519738924375,
+      "loss": 3.568,
       "step": 1500
     },
     {
       "epoch": 0.19727756954034326,
-      "grad_norm": 0.19324058294296265,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.0313,
+      "grad_norm": 0.6391732096672058,
+      "learning_rate": 0.0008697519738924375,
+      "loss": 3.5636,
       "step": 2000
     },
     {
       "epoch": 0.24659696192542907,
-      "grad_norm": 0.1767522692680359,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.0206,
+      "grad_norm": 0.8909493088722229,
+      "learning_rate": 0.0008697519738924375,
+      "loss": 3.5639,
       "step": 2500
     },
     {
       "epoch": 0.2959163543105149,
-      "grad_norm": 0.20005017518997192,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.0154,
+      "grad_norm": 0.8647511005401611,
+      "learning_rate": 0.0008697519738924375,
+      "loss": 3.5623,
       "step": 3000
     },
     {
       "epoch": 0.3452357466956007,
-      "grad_norm": 0.14517371356487274,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.0121,
+      "grad_norm": 0.9398127794265747,
+      "learning_rate": 0.0008697519738924375,
+      "loss": 3.5588,
       "step": 3500
     },
     {
       "epoch": 0.3945551390806865,
-      "grad_norm": 0.1346338540315628,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.0113,
+      "grad_norm": 0.9252232909202576,
+      "learning_rate": 0.0008697519738924375,
+      "loss": 3.5597,
       "step": 4000
     },
     {
       "epoch": 0.4438745314657723,
-      "grad_norm": 0.13754339516162872,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.0089,
+      "grad_norm": 0.9499356746673584,
+      "learning_rate": 0.0008697519738924375,
+      "loss": 3.5584,
       "step": 4500
     },
     {
       "epoch": 0.49319392385085814,
-      "grad_norm": 0.14780430495738983,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.0078,
+      "grad_norm": 0.8844195008277893,
+      "learning_rate": 0.0008697519738924375,
+      "loss": 3.5581,
       "step": 5000
     },
     {
       "epoch": 0.49319392385085814,
-      "eval_all_ligands_equal": 0.518338488731772,
-      "eval_e3_equal": 0.8000883782589483,
+      "eval_all_ligands_equal": 0.0,
+      "eval_e3_equal": 0.0,
       "eval_e3_graph_edit_distance": 9.999999999999999e+63,
       "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9925762262483429,
-      "eval_e3_heavy_atoms_difference": 0.48475475033141846,
-      "eval_e3_heavy_atoms_difference_norm": 0.011724540067924432,
+      "eval_e3_has_attachment_point(s)": 0.0,
+      "eval_e3_heavy_atoms_difference": 28.086433937251435,
+      "eval_e3_heavy_atoms_difference_norm": 1.0,
       "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9925762262483429,
-      "eval_has_all_attachment_points": 0.9901900132567388,
-      "eval_has_three_substructures": 0.9992929739284137,
-      "eval_heavy_atoms_difference": 4.0311091471498015,
-      "eval_heavy_atoms_difference_norm": 0.05126412535753748,
-      "eval_linker_equal": 0.7454706142288997,
+      "eval_e3_valid": 0.0,
+      "eval_has_all_attachment_points": 0.0,
+      "eval_has_three_substructures": 0.0,
+      "eval_heavy_atoms_difference": 75.67370746796288,
+      "eval_heavy_atoms_difference_norm": 1.0,
+      "eval_linker_equal": 0.0,
       "eval_linker_graph_edit_distance": 9.999999999999999e+63,
       "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.99584622182943,
-      "eval_linker_heavy_atoms_difference": 0.3673000441891295,
-      "eval_linker_heavy_atoms_difference_norm": -0.001600385700560306,
+      "eval_linker_has_attachment_point(s)": 0.0,
+      "eval_linker_heavy_atoms_difference": 16.430402121078213,
+      "eval_linker_heavy_atoms_difference_norm": 1.0,
       "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.99584622182943,
-      "eval_loss": 0.3114722669124603,
-      "eval_num_fragments": 2.999646486964207,
-      "eval_poi_equal": 0.7483870967741936,
+      "eval_linker_valid": 0.0,
+      "eval_loss": 3.5819449424743652,
+      "eval_num_fragments": 1.0035351303579318,
+      "eval_poi_equal": 0.0,
       "eval_poi_graph_edit_distance": 9.999999999999999e+63,
       "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9661511268228016,
-      "eval_poi_heavy_atoms_difference": 0.9739284136102518,
-      "eval_poi_heavy_atoms_difference_norm": 0.02216572405609002,
+      "eval_poi_has_attachment_point(s)": 0.0,
+      "eval_poi_heavy_atoms_difference": 31.15687140963323,
+      "eval_poi_heavy_atoms_difference_norm": 1.0,
       "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9661511268228016,
-      "eval_reassembly": 0.5254971277065842,
-      "eval_reassembly_nostereo": 0.5520989836500221,
-      "eval_runtime": 1537.7347,
-      "eval_samples_per_second": 7.358,
-      "eval_steps_per_second": 0.115,
+      "eval_poi_valid": 0.0,
+      "eval_reassembly": 0.0,
+      "eval_reassembly_nostereo": 0.0,
+      "eval_runtime": 5690.2792,
+      "eval_samples_per_second": 1.988,
+      "eval_steps_per_second": 0.031,
       "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9566062748563853,
+      "eval_valid": 0.0,
       "step": 5000
-    },
-    {
-      "epoch": 0.542513316235944,
-      "grad_norm": 0.12387096881866455,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.0077,
-      "step": 5500
-    },
-    {
-      "epoch": 0.5918327086210298,
-      "grad_norm": 0.10272622108459473,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.0067,
-      "step": 6000
-    },
-    {
-      "epoch": 0.6411521010061156,
-      "grad_norm": 0.07910972088575363,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.0057,
-      "step": 6500
-    },
-    {
-      "epoch": 0.6904714933912014,
-      "grad_norm": 0.0813135877251625,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.0055,
-      "step": 7000
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "grad_norm": 0.12076853960752487,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.0056,
-      "step": 7500
-    },
-    {
-      "epoch": 0.7397908857762873,
-      "eval_all_ligands_equal": 0.5376049491825011,
-      "eval_e3_equal": 0.8114891736632789,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9906319045514803,
-      "eval_e3_heavy_atoms_difference": 0.29924878479893946,
-      "eval_e3_heavy_atoms_difference_norm": 0.005876065486626146,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9906319045514803,
-      "eval_has_all_attachment_points": 0.9876270437472382,
-      "eval_has_three_substructures": 0.9985859478568272,
-      "eval_heavy_atoms_difference": 4.727706584180291,
-      "eval_heavy_atoms_difference_norm": 0.05947950876380095,
-      "eval_linker_equal": 0.7778170570039771,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9939019001325674,
-      "eval_linker_heavy_atoms_difference": 0.483429076447194,
-      "eval_linker_heavy_atoms_difference_norm": 0.011003293636156783,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9939019001325674,
-      "eval_loss": 0.3112676441669464,
-      "eval_num_fragments": 2.998585947856827,
-      "eval_poi_equal": 0.7544851966416262,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9611135660627486,
-      "eval_poi_heavy_atoms_difference": 1.4438356164383561,
-      "eval_poi_heavy_atoms_difference_norm": 0.037136503882227626,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9611135660627486,
-      "eval_reassembly": 0.5441449403446752,
-      "eval_reassembly_nostereo": 0.5645603181617322,
-      "eval_runtime": 1602.1132,
-      "eval_samples_per_second": 7.063,
-      "eval_steps_per_second": 0.11,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9501546619531596,
-      "step": 7500
-    },
-    {
-      "epoch": 0.789110278161373,
-      "grad_norm": 0.0852850005030632,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.005,
-      "step": 8000
-    },
-    {
-      "epoch": 0.8384296705464589,
-      "grad_norm": 0.08112958818674088,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.0045,
-      "step": 8500
-    },
-    {
-      "epoch": 0.8877490629315447,
-      "grad_norm": 0.09284210950136185,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.0039,
-      "step": 9000
-    },
-    {
-      "epoch": 0.9370684553166305,
-      "grad_norm": 0.08725526928901672,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.0039,
-      "step": 9500
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "grad_norm": 0.09762605279684067,
-      "learning_rate": 7.582125965096081e-05,
-      "loss": 0.0042,
-      "step": 10000
-    },
-    {
-      "epoch": 0.9863878477017163,
-      "eval_all_ligands_equal": 0.5585505965532479,
-      "eval_e3_equal": 0.8152010605391073,
-      "eval_e3_graph_edit_distance": 9.999999999999999e+63,
-      "eval_e3_graph_edit_distance_norm": 1.0,
-      "eval_e3_has_attachment_point(s)": 0.9901016349977906,
-      "eval_e3_heavy_atoms_difference": 0.46036235086168803,
-      "eval_e3_heavy_atoms_difference_norm": 0.008622936653764582,
-      "eval_e3_tanimoto_similarity": 0.0,
-      "eval_e3_valid": 0.9901016349977906,
-      "eval_has_all_attachment_points": 0.9880689350419797,
-      "eval_has_three_substructures": 0.9981440565620857,
-      "eval_heavy_atoms_difference": 3.860715863897481,
-      "eval_heavy_atoms_difference_norm": 0.049434358555264034,
-      "eval_linker_equal": 0.7879805567830314,
-      "eval_linker_graph_edit_distance": 9.999999999999999e+63,
-      "eval_linker_graph_edit_distance_norm": 1.0,
-      "eval_linker_has_attachment_point(s)": 0.9946973044631021,
-      "eval_linker_heavy_atoms_difference": 0.547945205479452,
-      "eval_linker_heavy_atoms_difference_norm": 0.016672268873901653,
-      "eval_linker_tanimoto_similarity": 0.0,
-      "eval_linker_valid": 0.9946973044631021,
-      "eval_loss": 0.3271462023258209,
-      "eval_num_fragments": 2.998497569597879,
-      "eval_poi_equal": 0.7714538223596995,
-      "eval_poi_graph_edit_distance": 9.999999999999999e+63,
-      "eval_poi_graph_edit_distance_norm": 1.0,
-      "eval_poi_has_attachment_point(s)": 0.9708351745470615,
-      "eval_poi_heavy_atoms_difference": 0.9135660627485639,
-      "eval_poi_heavy_atoms_difference_norm": 0.02195151202596944,
-      "eval_poi_tanimoto_similarity": 0.0,
-      "eval_poi_valid": 0.9708351745470615,
-      "eval_reassembly": 0.5670349094122846,
-      "eval_reassembly_nostereo": 0.5946089262041537,
-      "eval_runtime": 1538.8263,
-      "eval_samples_per_second": 7.353,
-      "eval_steps_per_second": 0.115,
-      "eval_tanimoto_similarity": 0.0,
-      "eval_valid": 0.9598762704374724,
-      "step": 10000
     }
   ],
   "logging_steps": 500,
@@ -287,18 +133,18 @@
         "should_evaluate": false,
         "should_log": false,
         "should_save": true,
-        "should_training_stop": true
+        "should_training_stop": false
       },
       "attributes": {}
     }
   },
-  "total_flos": 8.729673191023411e+16,
+  "total_flos": 4.364959997072179e+16,
   "train_batch_size": 128,
-  "trial_name": "trial-number=10-learning_rate=7.6e-05-warmup_ratio=0.040-min_lr=0.000-factor=0.940",
+  "trial_name": "trial-number=11-learning_rate=8.7e-04-warmup_ratio=0.080-min_lr=0.000-factor=0.280",
   "trial_params": {
-    "factor": 0.94,
-    "learning_rate": 7.582125965096081e-05,
-    "min_lr": 5.001256479531022e-11,
-    "warmup_ratio": 0.04
+    "factor": 0.28,
+    "learning_rate": 0.0008697519738924375,
+    "min_lr": 2.1479583400971326e-12,
+    "warmup_ratio": 0.08
   }
 }

last-checkpoint/training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b141933d3d59f747dba8d7362622a470e9dacf61df786a6eff32bf95c9982720
+oid sha256:dca0687c8354a27c0ca35e94394adbcca2789cd7dadea8da06b5552db1bd21f7
 size 7544